[clang] [llvm] [clang][MBD] set up module build daemon infrastructure (PR #67562)
Connor Sughrue via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 10 11:17:51 PDT 2024
https://github.com/cpsughrue updated https://github.com/llvm/llvm-project/pull/67562
>From 92bdc8227423d798101dffc3baec70d2951bb058 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sun, 9 Jul 2023 23:19:58 -0400
Subject: [PATCH 01/20] [clang][MBD] set up module build daemon infrastructure
The module build daemon (mbd) will serve as a cc1 tool that helps convert
implicit module command lines to explicit module command lines. A clang
invocation will check to see if a mbd exists, if not the invocation will spawn
one. After the mbd is up and running and a handshake has successfully been
carried out between the mbd and clang invocation the clang invocation will
share it's command line with the mbd. The mbd will then scan the translation
unit and build all modular dependencies before returning a modified cc1 command
line such that all arguments related to modules are converted from the
implicit option to their explicit option.
This commit sets up the basic mbd infrastructure. Including the ability to
spawn a mbd and carry out a handshake between the clang invocation and mbd.
RFC: https://discourse.llvm.org/t/rfc-modules-build-daemon-build-system-agnostic-support-for-explicitly-built-modules/71524
---
.../clang/Basic/DiagnosticFrontendKinds.td | 20 ++
clang/include/clang/Basic/DiagnosticGroups.td | 1 +
clang/include/clang/Driver/Options.td | 13 +
.../include/clang/Frontend/FrontendOptions.h | 8 +
.../Tooling/ModuleBuildDaemon/Frontend.h | 36 +++
.../Tooling/ModuleBuildDaemon/SocketSupport.h | 126 +++++++++
.../clang/Tooling/ModuleBuildDaemon/Utils.h | 58 ++++
clang/lib/Driver/ToolChains/Clang.cpp | 14 +-
clang/lib/Tooling/CMakeLists.txt | 1 +
.../Tooling/ModuleBuildDaemon/CMakeLists.txt | 9 +
.../Tooling/ModuleBuildDaemon/Frontend.cpp | 166 ++++++++++++
.../ModuleBuildDaemon/SocketSupport.cpp | 66 +++++
clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp | 49 ++++
clang/test/Driver/unknown-arg.c | 2 +-
clang/test/Frontend/warning-options.cpp | 2 +-
clang/test/ModuleBuildDaemon/daemon-launch.c | 18 ++
clang/test/ModuleBuildDaemon/handshake.c | 22 ++
clang/test/lit.cfg.py | 11 +
clang/tools/driver/CMakeLists.txt | 3 +
clang/tools/driver/cc1_main.cpp | 17 +-
clang/tools/driver/cc1modbuildd_main.cpp | 255 ++++++++++++++++++
clang/tools/driver/driver.cpp | 13 +-
clang/utils/kill_process.py | 86 ++++++
llvm/include/llvm/Support/raw_socket_stream.h | 9 +
llvm/lib/Support/raw_socket_stream.cpp | 24 ++
25 files changed, 1017 insertions(+), 12 deletions(-)
create mode 100644 clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h
create mode 100644 clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
create mode 100644 clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
create mode 100644 clang/test/ModuleBuildDaemon/daemon-launch.c
create mode 100644 clang/test/ModuleBuildDaemon/handshake.c
create mode 100644 clang/tools/driver/cc1modbuildd_main.cpp
create mode 100644 clang/utils/kill_process.py
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 14b08d4927ec5e..50e3aab765c7da 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -264,6 +264,26 @@ def err_test_module_file_extension_version : Error<
"test module file extension '%0' has different version (%1.%2) than expected "
"(%3.%4)">;
+// Module Build Daemon
+def err_basepath_length :
+ Error<"BasePath '%0' is longer then the max length of %1">,
+ DefaultFatal;
+def err_mbd_handshake :
+ Error<"Attempt to handshake with module build daemon has failed: %0">,
+ DefaultFatal;
+def err_mbd_connect :
+ Error<"Attempt to connect to module build daemon has failed: %0">,
+ DefaultFatal;
+def remark_mbd_spawn :
+ Remark<"Successfully spawned module build daemon">,
+ InGroup<ModuleBuildDaemon>;
+def remark_mbd_connection :
+ Remark<"Successfully connected to module build daemon at %0">,
+ InGroup<ModuleBuildDaemon>;
+def remark_mbd_handshake :
+ Remark<"Successfully completed handshake with module build daemon">,
+ InGroup<ModuleBuildDaemon>;
+
def warn_eagerly_load_for_standard_cplusplus_modules : Warning<
"the form '-fmodule-file=<BMI-path>' is deprecated for standard C++ named modules;"
"consider to use '-fmodule-file=<module-name>=<BMI-path>' instead">,
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 47747d8704b6c8..119b79a0315f7c 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -528,6 +528,7 @@ def MissingFieldInitializers : DiagGroup<"missing-field-initializers",
[MissingDesignatedFieldInitializers]>;
def ModuleLock : DiagGroup<"module-lock">;
def ModuleBuild : DiagGroup<"module-build">;
+def ModuleBuildDaemon : DiagGroup<"module-build-daemon">;
def ModuleImport : DiagGroup<"module-import">;
def ModuleConflict : DiagGroup<"module-conflict">;
def ModuleFileExtension : DiagGroup<"module-file-extension">;
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index f745e573eb2686..bae5f43f1f5ad0 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3015,6 +3015,19 @@ defm declspec : BoolOption<"f", "declspec",
NegFlag<SetFalse, [], [ClangOption], "Disallow">,
BothFlags<[], [ClangOption, CC1Option],
" __declspec as a keyword">>, Group<f_clang_Group>;
+
+def fmodule_build_daemon : Flag<["-"], "fmodule-build-daemon">, Group<f_Group>,
+ Flags<[NoXarchOption]>,
+ Visibility<[ClangOption, CC1Option]>,
+ HelpText<"Enables module build daemon functionality">,
+ MarshallingInfoFlag<FrontendOpts<"ModuleBuildDaemon">>;
+def fmodule_build_daemon_EQ : Joined<["-"], "fmodule-build-daemon=">, Group<f_Group>,
+ Flags<[NoXarchOption]>,
+ Visibility<[ClangOption, CC1Option]>,
+ HelpText<"Enables module build daemon functionality and defines the path to "
+ "where the module build daemon will write log and socket files">,
+ MarshallingInfoString<FrontendOpts<"ModuleBuildDaemonPath">>;
+
def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">, Group<i_Group>,
Flags<[]>, Visibility<[ClangOption, CC1Option]>,
MetaVarName<"<directory>">,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 5ee4d471670f48..7ea87acbecc230 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -403,6 +403,10 @@ class FrontendOptions {
/// Whether to emit symbol labels for testing in generated symbol graphs
LLVM_PREFERRED_TYPE(bool)
unsigned EmitPrettySymbolGraphs : 1;
+
+ /// Connect to module build daemon.
+ LLVM_PREFERRED_TYPE(bool)
+ unsigned ModuleBuildDaemon : 1;
CodeCompleteOptions CodeCompleteOpts;
@@ -489,6 +493,10 @@ class FrontendOptions {
/// The output file, if any.
std::string OutputFile;
+ /// If given, the path to the module build daemon's output files and socket
+ /// address
+ std::string ModuleBuildDaemonPath;
+
/// If given, the new suffix for fix-it rewritten files.
std::string FixItSuffix;
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h b/clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h
new file mode 100644
index 00000000000000..727738ef77b1bd
--- /dev/null
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h
@@ -0,0 +1,36 @@
+//===----------------------------- Frontend.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_FRONTEND_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_FRONTEND_H
+
+#include "clang/Frontend/CompilerInvocation.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+namespace clang::tooling::cc1modbuildd {
+
+llvm::Error attemptHandshake(llvm::raw_socket_stream &Client,
+ clang::DiagnosticsEngine &Diag);
+
+llvm::Error spawnModuleBuildDaemon(const clang::CompilerInvocation &Clang,
+ const char *Argv0,
+ clang::DiagnosticsEngine &Diag,
+ std::string BasePath);
+
+llvm::Expected<std::unique_ptr<llvm::raw_socket_stream>>
+getModuleBuildDaemon(const clang::CompilerInvocation &Clang, const char *Argv0,
+ clang::DiagnosticsEngine &Diag, llvm::StringRef BasePath);
+
+void spawnModuleBuildDaemonAndHandshake(const clang::CompilerInvocation &Clang,
+ const char *Argv0,
+ clang::DiagnosticsEngine &Diag);
+
+} // namespace clang::tooling::cc1modbuildd
+
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_CLIENT_H
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h b/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
new file mode 100644
index 00000000000000..94a10359c61471
--- /dev/null
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
@@ -0,0 +1,126 @@
+//===------------------------- SocketMsgSupport.h -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+namespace clang::tooling::cc1modbuildd {
+
+enum class ActionType { HANDSHAKE };
+enum class StatusType { REQUEST, SUCCESS, FAILURE };
+
+struct BaseMsg {
+ ActionType MsgAction;
+ StatusType MsgStatus;
+
+ BaseMsg() = default;
+ BaseMsg(ActionType Action, StatusType Status)
+ : MsgAction(Action), MsgStatus(Status) {}
+};
+
+struct HandshakeMsg : public BaseMsg {
+ HandshakeMsg() = default;
+ HandshakeMsg(ActionType Action, StatusType Status)
+ : BaseMsg(Action, Status) {}
+};
+
+llvm::Expected<std::unique_ptr<llvm::raw_socket_stream>>
+connectToSocket(llvm::StringRef SocketPath);
+llvm::Expected<std::string>
+readBufferFromSocket(llvm::raw_socket_stream &Socket);
+llvm::Error writeBufferToSocket(llvm::raw_socket_stream &Socket,
+ llvm::StringRef Buffer);
+
+template <typename T> std::string convertMsgStructToBuffer(T MsgStruct) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ std::string Buffer;
+ llvm::raw_string_ostream OS(Buffer);
+ llvm::yaml::Output YamlOut(OS);
+
+ // TODO confirm yaml::Output does not have any error messages
+ YamlOut << MsgStruct;
+
+ return Buffer;
+}
+
+template <typename T>
+llvm::Expected<T> convertBufferToMsgStruct(llvm::StringRef Buffer) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ T MsgStruct;
+ llvm::yaml::Input YamlIn(Buffer);
+ YamlIn >> MsgStruct;
+
+ // YamlIn.error() dumps an error message if there is one
+ if (YamlIn.error())
+ return llvm::make_error<llvm::StringError>(
+ "Syntax or semantic error during YAML parsing",
+ llvm::inconvertibleErrorCode());
+
+ return MsgStruct;
+}
+
+template <typename T>
+llvm::Expected<T> readMsgStructFromSocket(llvm::raw_socket_stream &Socket) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ llvm::Expected<std::string> MaybeBuffer = readBufferFromSocket(Socket);
+ if (!MaybeBuffer)
+ return std::move(MaybeBuffer.takeError());
+ std::string Buffer = std::move(*MaybeBuffer);
+
+ llvm::Expected<T> MaybeMsgStruct = convertBufferToMsgStruct<T>(Buffer);
+ if (!MaybeMsgStruct)
+ return std::move(MaybeMsgStruct.takeError());
+ return std::move(*MaybeMsgStruct);
+}
+
+template <typename T>
+llvm::Error writeMsgStructToSocket(llvm::raw_socket_stream &Socket,
+ T MsgStruct) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ std::string Buffer = convertMsgStructToBuffer(MsgStruct);
+ if (llvm::Error Err = writeBufferToSocket(Socket, Buffer))
+ return std::move(Err);
+ return llvm::Error::success();
+}
+
+} // namespace clang::tooling::cc1modbuildd
+
+namespace cc1mod = clang::tooling::cc1modbuildd;
+
+template <> struct llvm::yaml::ScalarEnumerationTraits<cc1mod::StatusType> {
+ static void enumeration(IO &Io, cc1mod::StatusType &Value) {
+ Io.enumCase(Value, "REQUEST", cc1mod::StatusType::REQUEST);
+ Io.enumCase(Value, "SUCCESS", cc1mod::StatusType::SUCCESS);
+ Io.enumCase(Value, "FAILURE", cc1mod::StatusType::FAILURE);
+ }
+};
+
+template <> struct llvm::yaml::ScalarEnumerationTraits<cc1mod::ActionType> {
+ static void enumeration(IO &Io, cc1mod::ActionType &Value) {
+ Io.enumCase(Value, "HANDSHAKE", cc1mod::ActionType::HANDSHAKE);
+ }
+};
+
+template <> struct llvm::yaml::MappingTraits<cc1mod::HandshakeMsg> {
+ static void mapping(IO &Io, cc1mod::HandshakeMsg &Info) {
+ Io.mapRequired("Action", Info.MsgAction);
+ Io.mapRequired("Status", Info.MsgStatus);
+ }
+};
+
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
new file mode 100644
index 00000000000000..77fa4aee1e2ca9
--- /dev/null
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
@@ -0,0 +1,58 @@
+//===------------------------------ Utils.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Functions required by both the frontend and the module build daemon
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_UTILS_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_UTILS_H
+
+#include "llvm/Support/Error.h"
+
+#include <string>
+
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+// winsock2.h must be included before afunix.h
+// clang-format off
+#include <winsock2.h>
+#include <afunix.h>
+// clang-format on
+#else
+#include <sys/un.h>
+#endif
+
+namespace clang::tooling::cc1modbuildd {
+
+constexpr std::string_view SOCKET_FILE_NAME = "mbd.sock";
+constexpr std::string_view STDOUT_FILE_NAME = "mbd.out";
+constexpr std::string_view STDERR_FILE_NAME = "mbd.err";
+constexpr std::string_view MODULE_BUILD_DAEMON_FLAG = "-cc1modbuildd";
+
+// A llvm::raw_socket_stream uses sockaddr_un
+constexpr size_t SOCKET_ADDR_MAX_LENGTH = sizeof(sockaddr_un::sun_path);
+
+constexpr size_t BASEPATH_MAX_LENGTH =
+ SOCKET_ADDR_MAX_LENGTH - SOCKET_FILE_NAME.length();
+
+// How long should the module build daemon sit ideal before exiting
+constexpr int TimeoutSec = 15;
+
+// Get a temprary location where the daemon can store log files and a socket
+// address. Of the format /tmp/clang-<BLAKE3HashOfClangFullVersion>/
+std::string getBasePath();
+
+// Check if the user provided BasePath is short enough
+bool validBasePathLength(llvm::StringRef);
+
+} // namespace clang::tooling::cc1modbuildd
+
+#endif
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 766a9b91e3c0ad..e0fc45aa35199e 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-
#include "Clang.h"
#include "AMDGPU.h"
#include "Arch/AArch64.h"
@@ -3865,6 +3864,19 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
bool HaveStdCXXModules = IsCXX && HaveStd20;
bool HaveModules = HaveStdCXXModules;
+ // -fmodule-build-daemon enables module build daemon functionality
+ if (Args.hasArg(options::OPT_fmodule_build_daemon))
+ Args.AddLastArg(CmdArgs, options::OPT_fmodule_build_daemon);
+
+ // by default module build daemon socket address and output files are saved
+ // under /tmp/ but that can be overridden by providing the
+ // -fmodule-build-daemon=<path> flag
+ if (Arg *A = Args.getLastArg(options::OPT_fmodule_build_daemon_EQ)) {
+ CmdArgs.push_back(
+ Args.MakeArgString(Twine("-fmodule-build-daemon=") + A->getValue()));
+ CmdArgs.push_back("-fmodule-build-daemon");
+ }
+
// -fmodules enables the use of precompiled modules (off by default).
// Users can pass -fno-cxx-modules to turn off modules support for
// C++/Objective-C++ programs.
diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt
index 8b4ab0e2129649..08821c105cefb0 100644
--- a/clang/lib/Tooling/CMakeLists.txt
+++ b/clang/lib/Tooling/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(DumpTool)
add_subdirectory(Syntax)
add_subdirectory(DependencyScanning)
add_subdirectory(Transformer)
+add_subdirectory(ModuleBuildDaemon)
# Replace the last lib component of the current binary directory with include
string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE)
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt b/clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
new file mode 100644
index 00000000000000..c043557206ef9e
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
@@ -0,0 +1,9 @@
+set(LLVM_LINK_COMPONENTS
+ Support
+ )
+
+add_clang_library(clangModuleBuildDaemon
+ Frontend.cpp
+ SocketSupport.cpp
+ Utils.cpp
+ )
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
new file mode 100644
index 00000000000000..cf73e27f246bba
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -0,0 +1,166 @@
+//===---------------------------- Frontend.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/Frontend.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h"
+#include "clang/Tooling/ModuleBuildDaemon/Utils.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Program.h"
+
+#include <cerrno>
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <optional>
+#include <string>
+#include <thread>
+
+using namespace llvm;
+
+namespace clang::tooling::cc1modbuildd {
+
+llvm::Error attemptHandshake(raw_socket_stream &Client,
+ DiagnosticsEngine &Diag) {
+
+ // Send HandshakeMsg to module build daemon
+ HandshakeMsg Request{ActionType::HANDSHAKE, StatusType::REQUEST};
+ if (llvm::Error Err = writeMsgStructToSocket(Client, Request))
+ return std::move(Err);
+
+ // Read response from module build daemon
+ Expected<HandshakeMsg> MaybeResponse =
+ readMsgStructFromSocket<HandshakeMsg>(Client);
+ if (!MaybeResponse) {
+ return std::move(MaybeResponse.takeError());
+ }
+ HandshakeMsg Response = std::move(*MaybeResponse);
+
+ assert(Response.MsgAction == ActionType::HANDSHAKE &&
+ "The response ActionType should only ever be HANDSHAKE");
+
+ if (Response.MsgStatus == StatusType::SUCCESS) {
+ return llvm::Error::success();
+ }
+
+ return llvm::make_error<StringError>(
+ "Received handshake response 'FAILURE' from module build daemon",
+ std::make_error_code(std::errc::operation_not_permitted));
+}
+
+llvm::Error spawnModuleBuildDaemon(const CompilerInvocation &Clang,
+ const char *Argv0, DiagnosticsEngine &Diag,
+ std::string BasePath) {
+
+ std::vector<StringRef> Args = {Argv0, MODULE_BUILD_DAEMON_FLAG};
+ if (!Clang.getFrontendOpts().ModuleBuildDaemonPath.empty())
+ Args.push_back(BasePath.c_str());
+
+ std::string ErrorBuffer;
+ llvm::sys::ExecuteNoWait(Argv0, Args, std::nullopt, {}, 0, &ErrorBuffer,
+ nullptr, nullptr, /*DetachProcess*/ true);
+
+ // llvm::sys::ExecuteNoWait can fail for a variety of reasons which can't be
+ // generalized to one error code
+ if (!ErrorBuffer.empty())
+ return llvm::make_error<StringError>(ErrorBuffer, inconvertibleErrorCode());
+
+ Diag.Report(diag::remark_mbd_spawn);
+ return llvm::Error::success();
+}
+
+Expected<std::unique_ptr<raw_socket_stream>>
+getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
+ DiagnosticsEngine &Diag, StringRef BasePath) {
+
+ SmallString<128> SocketPath = BasePath;
+ llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+
+ if (llvm::sys::fs::exists(SocketPath)) {
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ connectToSocket(SocketPath);
+ if (MaybeClient) {
+ return std::move(*MaybeClient);
+ }
+ consumeError(MaybeClient.takeError());
+ }
+
+ if (llvm::Error Err =
+ spawnModuleBuildDaemon(Clang, Argv0, Diag, BasePath.str()))
+ return std::move(Err);
+
+ constexpr unsigned int MICROSEC_IN_SEC = 1000000;
+ constexpr unsigned int MAX_WAIT_TIME = 30 * MICROSEC_IN_SEC;
+ unsigned int CumulativeTime = 0;
+ unsigned int WaitTime = 10;
+
+ while (CumulativeTime <= MAX_WAIT_TIME) {
+ // Wait a bit then check to see if the module build daemon has initialized
+ std::this_thread::sleep_for(std::chrono::microseconds(WaitTime));
+
+ if (llvm::sys::fs::exists(SocketPath)) {
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ connectToSocket(SocketPath);
+ if (MaybeClient) {
+ Diag.Report(diag::remark_mbd_connection) << SocketPath;
+ return std::move(*MaybeClient);
+ }
+ consumeError(MaybeClient.takeError());
+ }
+
+ CumulativeTime += WaitTime;
+ WaitTime = WaitTime * 2;
+ }
+
+ // After waiting around 30 seconds give up and return an error
+ return llvm::make_error<StringError>(
+ "Max wait time exceeded: ",
+ std::make_error_code(std::errc::no_such_process));
+}
+
+void spawnModuleBuildDaemonAndHandshake(const CompilerInvocation &Clang,
+ const char *Argv0,
+ DiagnosticsEngine &Diag) {
+
+ // The module build daemon stores all output files and its socket address
+ // under BasePath. Either set BasePath to a user provided option or create an
+ // appropriate BasePath based on the BLAKE3 hash of the full clang version
+ std::string BasePath;
+ if (Clang.getFrontendOpts().ModuleBuildDaemonPath.empty())
+ BasePath = getBasePath();
+ else {
+ // Get user provided BasePath and confirm it is short enough
+ BasePath = Clang.getFrontendOpts().ModuleBuildDaemonPath;
+ if (!validBasePathLength(BasePath)) {
+ Diag.Report(diag::err_basepath_length) << BasePath << BASEPATH_MAX_LENGTH;
+ return;
+ }
+ }
+
+ // If module build daemon does not exist spawn module build daemon
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ getModuleBuildDaemon(Clang, Argv0, Diag, BasePath);
+ if (!MaybeClient) {
+ Diag.Report(diag::err_mbd_connect) << MaybeClient.takeError();
+ return;
+ }
+ raw_socket_stream &Client = **MaybeClient;
+ if (llvm::Error HandshakeErr = attemptHandshake(Client, Diag)) {
+ Diag.Report(diag::err_mbd_handshake) << std::move(HandshakeErr);
+ return;
+ }
+
+ Diag.Report(diag::remark_mbd_handshake);
+ return;
+}
+
+} // namespace clang::tooling::cc1modbuildd
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp b/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
new file mode 100644
index 00000000000000..08ddfff8c7f387
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
@@ -0,0 +1,66 @@
+//===------------------------ SocketMsgSupport.cpp ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+#include <memory>
+#include <string>
+
+using namespace llvm;
+
+namespace clang::tooling::cc1modbuildd {
+
+Expected<std::unique_ptr<raw_socket_stream>>
+connectToSocket(StringRef SocketPath) {
+
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ raw_socket_stream::createConnectedUnix(SocketPath);
+ if (!MaybeClient)
+ return std::move(MaybeClient.takeError());
+
+ return std::move(*MaybeClient);
+}
+
+Expected<std::string> readBufferFromSocket(raw_socket_stream &Socket) {
+
+ constexpr unsigned short MAX_BUFFER = 4096;
+ char Buffer[MAX_BUFFER];
+ std::string ReturnBuffer;
+
+ ssize_t n = 0;
+ while ((n = Socket.read(Buffer, MAX_BUFFER)) > 0) {
+ ReturnBuffer.append(Buffer, n);
+ // Read until \n... encountered which is the last line of a YAML document
+ if (ReturnBuffer.find("\n...") != std::string::npos)
+ break;
+ }
+
+ if (Socket.has_error()) {
+ std::error_code EC = Socket.error();
+ Socket.clear_error();
+ return make_error<StringError>("Failed socket read: ", EC);
+ }
+ return ReturnBuffer;
+}
+
+Error writeBufferToSocket(raw_socket_stream &Socket, StringRef Buffer) {
+ Socket << Buffer;
+
+ if (Socket.has_error()) {
+ std::error_code EC = Socket.error();
+ Socket.clear_error();
+ return make_error<StringError>("Failed socket write: ", EC);
+ }
+
+ Socket.flush();
+ return Error::success();
+}
+
+} // namespace clang::tooling::cc1modbuildd
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
new file mode 100644
index 00000000000000..93fc14c4ba23dc
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
@@ -0,0 +1,49 @@
+//===------------------------------ Utils.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/Utils.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/BLAKE3.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Path.h"
+
+#include <string>
+
+using namespace llvm;
+
+namespace clang::tooling::cc1modbuildd {
+
+std::string getBasePath() {
+ llvm::BLAKE3 Hash;
+ Hash.update(clang::getClangFullVersion());
+ auto HashResult = Hash.final<sizeof(uint64_t)>();
+ uint64_t HashValue =
+ llvm::support::endian::read<uint64_t, llvm::endianness::native>(
+ HashResult.data());
+ std::string Key = toString(llvm::APInt(64, HashValue), 36, /*Signed*/ false);
+
+ // Set paths
+ SmallString<128> BasePath;
+ llvm::sys::path::system_temp_directory(/*erasedOnReboot*/ true, BasePath);
+ llvm::sys::path::append(BasePath, "clang-" + Key);
+ return BasePath.c_str();
+}
+
+bool validBasePathLength(llvm::StringRef Address) {
+ // Confirm that the user provided BasePath is short enough to allow the socket
+ // address to fit within the alloted space
+ if (Address.str().length() > BASEPATH_MAX_LENGTH) {
+ return false;
+ }
+ return true;
+}
+
+} // namespace clang::tooling::cc1modbuildd
\ No newline at end of file
diff --git a/clang/test/Driver/unknown-arg.c b/clang/test/Driver/unknown-arg.c
index 26032a407f4131..3a22b824adc5c7 100644
--- a/clang/test/Driver/unknown-arg.c
+++ b/clang/test/Driver/unknown-arg.c
@@ -59,7 +59,7 @@
// SILENT-NOT: warning:
// CC1AS-DID-YOU-MEAN: error: unknown argument '-hell'; did you mean '-help'?
// CC1AS-DID-YOU-MEAN: error: unknown argument '--version'; did you mean '-version'?
-// UNKNOWN-INTEGRATED: error: unknown integrated tool '-cc1asphalt'. Valid tools include '-cc1', '-cc1as' and '-cc1gen-reproducer'.
+// UNKNOWN-INTEGRATED: error: unknown integrated tool '-cc1asphalt'. Valid tools include '-cc1', '-cc1as', '-cc1gen-reproducer', and '-cc1modbuildd'.
// RUN: %clang -S %s -o %t.s -Wunknown-to-clang-option 2>&1 | FileCheck --check-prefix=IGNORED %s
diff --git a/clang/test/Frontend/warning-options.cpp b/clang/test/Frontend/warning-options.cpp
index 444733c8b7f365..42a7109b8d48c5 100644
--- a/clang/test/Frontend/warning-options.cpp
+++ b/clang/test/Frontend/warning-options.cpp
@@ -3,6 +3,6 @@
// CHECK: unknown warning option '-Wmonkey'
// CHECK: unknown warning option '-Wno-monkey'
// CHECK: unknown warning option '-Wno-unused-command-line-arguments'; did you mean '-Wno-unused-command-line-argument'?
-// CHECK: unknown warning option '-Wmodule-build'; did you mean '-Wmodule-conflict'?
+// CHECK: unknown warning option '-Wmodule-build'
// CHECK-NEXT: unknown -Werror warning specifier: '-Werror-vla'
// CHECK: unknown remark option '-Rmodule-built'; did you mean '-Rmodule-build'?
diff --git a/clang/test/ModuleBuildDaemon/daemon-launch.c b/clang/test/ModuleBuildDaemon/daemon-launch.c
new file mode 100644
index 00000000000000..2bfea6aec9dccd
--- /dev/null
+++ b/clang/test/ModuleBuildDaemon/daemon-launch.c
@@ -0,0 +1,18 @@
+// Check that the module build daemon can create a unix socket
+
+// RUN: rm -rf mbd-launch %t
+
+// timeout should exit with status 124 which is treated as a failure by lit on
+// windows. Ideally we would be like to check the exit code and only return true
+// if it equals 124 but lit does not support global bash sysmbols like $?
+
+// RUN: timeout --signal=SIGTERM 2 %clang -cc1modbuildd mbd-launch -v || true
+// RUN: cat mbd-launch/mbd.out | sed 's:\\\\\?:/:g' | FileCheck %s
+
+// CHECK: mbd created and binded to socket at: mbd-launch/mbd.sock
+
+// Make sure mbd.sock does not exist
+// RUN: [ ! -f "mbd-launch/mbd.sock" ] && true || false
+
+// Make sure mbd.err is empty
+// RUN: [ ! -s "mbd-launch/mbd.err" ] && true || false
diff --git a/clang/test/ModuleBuildDaemon/handshake.c b/clang/test/ModuleBuildDaemon/handshake.c
new file mode 100644
index 00000000000000..059ecf6fae188c
--- /dev/null
+++ b/clang/test/ModuleBuildDaemon/handshake.c
@@ -0,0 +1,22 @@
+// Check that a clang invocation can spawn and handshake with a module build daemon
+
+// RUN: %kill-process "-cc1modbuildd mbd-handshake"
+// RUN: rm -rf mbd-handshake %t
+// RUN: split-file %s %t
+
+//--- main.c
+int main() {return 0;}
+
+// RUN: %clang -fmodule-build-daemon=mbd-handshake -Rmodule-build-daemon %t/main.c &> %t/output-new || true
+// RUN: %clang -fmodule-build-daemon=mbd-handshake -Rmodule-build-daemon %t/main.c &> %t/output-existing || true
+// RUN: %kill-process "-cc1modbuildd mbd-handshake"
+
+// RUN: cat %t/output-new | sed 's:\\\\\?:/:g' | FileCheck %s
+// RUN: cat %t/output-existing | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-EXIST
+
+// CHECK: remark: Successfully spawned module build daemon [-Rmodule-build-daemon]
+// CHECK: remark: Successfully connected to module build daemon at mbd-handshake/mbd.sock [-Rmodule-build-daemon]
+// CHECK: remark: Successfully completed handshake with module build daemon [-Rmodule-build-daemon]
+
+// Check that a clang invocation can handshake with an existing module build daemon
+// CHECK-EXIST: remark: Successfully completed handshake with module build daemon [-Rmodule-build-daemon]
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index e5630a07424c7c..19d225ffff82fc 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -205,6 +205,17 @@ def have_host_clang_repl_cuda():
)
)
+config.substitutions.append(
+ (
+ "%kill-process",
+ '"%s" %s'
+ % (
+ config.python_executable,
+ os.path.join(config.clang_src_dir, "utils", "kill_process.py"),
+ ),
+ )
+)
+
config.substitutions.append(("%host_cc", config.host_cc))
config.substitutions.append(("%host_cxx", config.host_cxx))
diff --git a/clang/tools/driver/CMakeLists.txt b/clang/tools/driver/CMakeLists.txt
index d70b92b0984e52..e44475b58b8ac4 100644
--- a/clang/tools/driver/CMakeLists.txt
+++ b/clang/tools/driver/CMakeLists.txt
@@ -28,6 +28,7 @@ add_clang_tool(clang
cc1_main.cpp
cc1as_main.cpp
cc1gen_reproducer_main.cpp
+ cc1modbuildd_main.cpp
DEPENDS
intrinsics_gen
@@ -39,9 +40,11 @@ clang_target_link_libraries(clang
PRIVATE
clangBasic
clangCodeGen
+ clangDependencyScanning
clangDriver
clangFrontend
clangFrontendTool
+ clangModuleBuildDaemon
clangSerialization
)
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index b5c6be3c557bb3..d93c4e46429554 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -25,6 +25,7 @@
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/FrontendTool/Utils.h"
+#include "clang/Tooling/ModuleBuildDaemon/Frontend.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LinkAllPasses.h"
@@ -63,7 +64,7 @@ using namespace llvm::opt;
static void LLVMErrorHandler(void *UserData, const char *Message,
bool GenCrashDiag) {
- DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
+ DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine *>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
@@ -139,7 +140,7 @@ static int PrintSupportedExtensions(std::string TargetStr) {
const llvm::Triple &MachineTriple = TheTargetMachine->getTargetTriple();
const llvm::MCSubtargetInfo *MCInfo = TheTargetMachine->getMCSubtargetInfo();
const llvm::ArrayRef<llvm::SubtargetFeatureKV> Features =
- MCInfo->getAllProcessorFeatures();
+ MCInfo->getAllProcessorFeatures();
llvm::StringMap<llvm::StringRef> DescMap;
for (const llvm::SubtargetFeatureKV &feature : Features)
@@ -208,7 +209,7 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
Clang->getHeaderSearchOpts().ResourceDir.empty())
Clang->getHeaderSearchOpts().ResourceDir =
- CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
+ CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
// Create the actual diagnostics engine.
Clang->createDiagnostics();
@@ -217,8 +218,8 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
- llvm::install_fatal_error_handler(LLVMErrorHandler,
- static_cast<void*>(&Clang->getDiagnostics()));
+ llvm::install_fatal_error_handler(
+ LLVMErrorHandler, static_cast<void *>(&Clang->getDiagnostics()));
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
if (!Success) {
@@ -226,6 +227,12 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
return 1;
}
+ // Handle module build daemon functionality if enabled
+ if (Clang->getFrontendOpts().ModuleBuildDaemon) {
+ clang::tooling::cc1modbuildd::spawnModuleBuildDaemonAndHandshake(
+ Clang->getInvocation(), Argv0, Clang->getDiagnostics());
+ }
+
// Execute the frontend actions.
{
llvm::TimeTraceScope TimeScope("ExecuteCompiler");
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
new file mode 100644
index 00000000000000..e62bdd476abe83
--- /dev/null
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -0,0 +1,255 @@
+//===------- cc1modbuildd_main.cpp - Clang CC1 Module Build Daemon --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h"
+#include "clang/Tooling/ModuleBuildDaemon/Utils.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
+
+#include <csignal>
+#include <cstdbool>
+#include <fstream>
+#include <optional>
+#include <string>
+#include <system_error>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+using namespace llvm;
+using namespace clang::tooling::cc1modbuildd;
+
+// Create unbuffered STDOUT stream so that any logging done by the module build
+// daemon can be viewed without having to terminate the process
+static raw_fd_ostream &unbuff_outs() {
+ static raw_fd_ostream S(fileno(stdout), false, true);
+ return S;
+}
+
+static bool VerboseLog = false;
+static void verboseLog(const llvm::Twine &message) {
+ if (VerboseLog) {
+ unbuff_outs() << message << '\n';
+ }
+}
+
+namespace {
+
+class ModuleBuildDaemonServer {
+public:
+ SmallString<256> SocketPath;
+ SmallString<256> STDERR;
+ SmallString<256> STDOUT;
+
+ ModuleBuildDaemonServer(StringRef Path)
+ : SocketPath(Path), STDERR(Path), STDOUT(Path) {
+ llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+ llvm::sys::path::append(STDOUT, STDOUT_FILE_NAME);
+ llvm::sys::path::append(STDERR, STDERR_FILE_NAME);
+ }
+
+ ~ModuleBuildDaemonServer() { shutdownDaemon(); }
+
+ int setupDaemonEnv();
+ int createDaemonSocket();
+ int listenForClients();
+
+ static void handleConnection(std::shared_ptr<raw_socket_stream> Connection);
+
+ // TODO: modify so when shutdownDaemon is called the daemon stops accepting
+ // new client connections and waits for all existing client connections to
+ // terminate before closing the file descriptor and exiting
+ void shutdownDaemon() {
+ RunServiceLoop = false;
+ if (ServerListener.has_value())
+ ServerListener.value().shutdown();
+ }
+
+private:
+ std::atomic<bool> RunServiceLoop = true;
+ std::optional<llvm::ListeningSocket> ServerListener;
+};
+
+// Used to handle signals
+ModuleBuildDaemonServer *DaemonPtr = nullptr;
+void handleSignal(int) {
+ DaemonPtr->shutdownDaemon();
+ exit(EXIT_SUCCESS);
+}
+} // namespace
+
+// Sets up file descriptors and signals for module build daemon
+int ModuleBuildDaemonServer::setupDaemonEnv() {
+
+#ifdef _WIN32
+ freopen("NUL", "r", stdin);
+#else
+ close(STDIN_FILENO);
+#endif
+
+ freopen(STDOUT.c_str(), "a", stdout);
+ freopen(STDERR.c_str(), "a", stderr);
+
+ if (std::signal(SIGTERM, handleSignal) == SIG_ERR) {
+ errs() << "failed to handle SIGTERM" << '\n';
+ exit(EXIT_FAILURE);
+ }
+
+ if (std::signal(SIGINT, handleSignal) == SIG_ERR) {
+ errs() << "failed to handle SIGINT" << '\n';
+ exit(EXIT_FAILURE);
+ }
+
+// TODO: Figure out how to do this on windows
+#ifdef SIGHUP
+ if (::signal(SIGHUP, SIG_IGN) == SIG_ERR) {
+ errs() << "failed to handle SIGHUP" << '\n';
+ exit(EXIT_FAILURE);
+ }
+#endif
+
+ return EXIT_SUCCESS;
+}
+
+// Creates unix socket for IPC with frontends
+int ModuleBuildDaemonServer::createDaemonSocket() {
+
+ Expected<ListeningSocket> MaybeServerListener =
+ llvm::ListeningSocket::createUnix(SocketPath);
+
+ if (llvm::Error Err = MaybeServerListener.takeError()) {
+ llvm::handleAllErrors(std::move(Err), [&](const llvm::StringError &SE) {
+ std::error_code EC = SE.convertToErrorCode();
+ // Exit successfully if the socket address is already in use. When
+ // translation units are compiled in parallel, until the socket file is
+ // created, all clang invocations will try to spawn a module build daemon.
+#ifdef _WIN32
+ if (EC.value() == WSAEADDRINUSE) {
+#else
+ if (EC == std::errc::address_in_use) {
+#endif
+ exit(EXIT_SUCCESS);
+ } else {
+ llvm::errs() << "MBD failed to create unix socket: " << SE.message()
+ << EC.message() << '\n';
+ exit(EXIT_FAILURE);
+ }
+ });
+ }
+
+ verboseLog("mbd created and binded to socket at: " + SocketPath);
+ ServerListener.emplace(std::move(*MaybeServerListener));
+ return 0;
+}
+
+// Function submitted to thread pool with each frontend connection. Not
+// responsible for closing frontend socket connections
+void ModuleBuildDaemonServer::handleConnection(
+ std::shared_ptr<llvm::raw_socket_stream> MovableConnection) {
+
+ llvm::raw_socket_stream &Connection = *MovableConnection;
+
+ // Read request from frontend
+ Expected<HandshakeMsg> MaybeHandshakeMsg =
+ readMsgStructFromSocket<HandshakeMsg>(Connection);
+ if (!MaybeHandshakeMsg) {
+ errs() << "MBD failed to read frontend request: "
+ << llvm::toString(MaybeHandshakeMsg.takeError()) << '\n';
+ return;
+ }
+
+ // Send response to frontend
+ HandshakeMsg Msg(ActionType::HANDSHAKE, StatusType::SUCCESS);
+ if (llvm::Error WriteErr = writeMsgStructToSocket(Connection, Msg)) {
+ errs() << "MBD failed to respond to frontend request: "
+ << llvm::toString(std::move(WriteErr)) << '\n';
+ return;
+ }
+ return;
+}
+
+int ModuleBuildDaemonServer::listenForClients() {
+
+ llvm::ThreadPool Pool;
+
+ while (RunServiceLoop) {
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeConnection =
+ ServerListener.value().accept({TimeoutSec, 0});
+
+ if (llvm::Error Err = MaybeConnection.takeError()) {
+
+ llvm::handleAllErrors(std::move(Err), [&](const llvm::StringError &SE) {
+ std::error_code EC = SE.convertToErrorCode();
+ if (EC == std::errc::timed_out)
+ RunServiceLoop = false;
+ else
+ errs() << "MBD failed to accept incoming connection: "
+ << SE.getMessage() << ": " << EC.message() << '\n';
+ });
+
+ continue;
+ }
+
+ std::shared_ptr<raw_socket_stream> Connection(std::move(*MaybeConnection));
+ Pool.async(handleConnection, Connection);
+ }
+ return 0;
+}
+
+// Module build daemon is spawned with the following command line:
+//
+// clang -cc1modbuildd [<path>] [-v]
+//
+// OPTIONS
+// <path>
+// Specifies the path to all the files created by the module build daemon.
+// If provided, <path> should immediately follow -cc1modbuildd.
+//
+// -v
+// Provides verbose debug information.
+//
+// NOTES
+// The arguments <path> and -v are optional. If <path> is not provided then
+// BasePath will be /tmp/clang-<BLAKE3HashOfClangFullVersion>
+//
+int cc1modbuildd_main(ArrayRef<const char *> Argv) {
+
+ // -cc1modbuildd is sliced away when Argv is pased to cc1modbuildd_main
+ if (find(Argv, StringRef("-v")) != Argv.end())
+ VerboseLog = true;
+
+ std::string BasePath;
+ // If an argument exists and it is not -v then it must be a BasePath
+ if (!Argv.empty() && strcmp(Argv[0], "-v") != 0)
+ BasePath = Argv[0];
+ else
+ BasePath = getBasePath();
+
+ if (!validBasePathLength(BasePath)) {
+ errs() << "BasePath '" << BasePath << "' is longer then the max length of "
+ << std::to_string(BASEPATH_MAX_LENGTH) << '\n';
+ return 1;
+ }
+
+ llvm::sys::fs::create_directories(BasePath);
+ ModuleBuildDaemonServer Daemon(BasePath);
+
+ // Used to handle signals
+ DaemonPtr = &Daemon;
+
+ Daemon.setupDaemonEnv();
+ Daemon.createDaemonSocket();
+ Daemon.listenForClients();
+
+ return EXIT_SUCCESS;
+}
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index 83b5bbb71f5212..54ff34e7426b06 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -84,6 +84,7 @@ extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
const char *Argv0, void *MainAddr,
const llvm::ToolContext &);
+extern int cc1modbuildd_main(ArrayRef<const char *> Argv);
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
SmallVectorImpl<const char *> &ArgVector,
@@ -218,10 +219,14 @@ static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV,
if (Tool == "-cc1gen-reproducer")
return cc1gen_reproducer_main(ArrayRef(ArgV).slice(2), ArgV[0],
GetExecutablePathVP, ToolContext);
- // Reject unknown tools.
- llvm::errs()
- << "error: unknown integrated tool '" << Tool << "'. "
- << "Valid tools include '-cc1', '-cc1as' and '-cc1gen-reproducer'.\n";
+ if (Tool == "-cc1modbuildd") {
+ return cc1modbuildd_main(ArrayRef(ArgV).slice(2));
+ }
+
+ // Reject unknown tools
+ llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
+ << "Valid tools include '-cc1', '-cc1as', '-cc1gen-reproducer', "
+ "and '-cc1modbuildd'.\n";
return 1;
}
diff --git a/clang/utils/kill_process.py b/clang/utils/kill_process.py
new file mode 100644
index 00000000000000..68d0e4ed8dc629
--- /dev/null
+++ b/clang/utils/kill_process.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+# Kills all processes whos command line match provided pattern
+
+import os
+import signal
+import subprocess
+import sys
+import platform
+
+
+def windows_impl(search_pattern):
+ cmd = "wmic process get ProcessId,CommandLine"
+ output = subprocess.check_output(cmd, shell=True, universal_newlines=True)
+ lines = output.split("\n")
+
+ PIDs = []
+
+ for line in lines:
+ if "clang" in line and "kill_process" not in line and "llvm-lit" not in line:
+
+ # creates something like ['..\\llvm-project\\build\\bin\\clang.exe -cc1modbuildd mbd-launch -v', '16260']
+ parts = line.rsplit(None, 1)
+
+ if len(parts) == 2:
+ command_line, pid = parts
+ if search_pattern in command_line:
+ PIDs.append(int(pid))
+
+ return PIDs
+
+
+def unix_impl(search_pattern):
+ ps_output = subprocess.check_output(["ps", "aux"], universal_newlines=True)
+ ps_lines = ps_output.split("\n")
+
+ PIDs = []
+
+ for line in ps_lines:
+ if "clang" in line and "kill_process" not in line and "llvm-lit" not in line:
+ ps_line_components = line.split()
+
+ # output of ps aux
+ # 0 1 2 3 4 5 6 7 8 9 10
+ # USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
+
+ # skip line without COMMAND
+ if len(ps_line_components) < 11:
+ continue
+
+ command_line = " ".join(ps_line_components[10:])
+ if search_pattern in command_line:
+ PIDs.append(int(ps_line_components[1]))
+
+ return PIDs
+
+
+def main():
+ if len(sys.argv) == 1:
+ sys.stderr.write("Error: no search pattern provided\n")
+ sys.exit(1)
+
+ search_pattern = sys.argv[1]
+ sys.stdout.write(
+ f"Searching for process with pattern '{search_pattern}' in command line\n"
+ )
+
+ system_name = platform.system()
+
+ PIDs = []
+ if system_name == "Windows":
+ PIDs = windows_impl(search_pattern)
+ else:
+ PIDs = unix_impl(search_pattern)
+
+ if len(PIDs) == 0:
+ sys.stdout.write("No processes matching search pattern were found\n")
+ return
+
+ sys.stdout.write(f"Killing PIDs {PIDs}\n")
+ for PID in PIDs:
+ os.kill(PID, signal.SIGTERM)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/llvm/include/llvm/Support/raw_socket_stream.h b/llvm/include/llvm/Support/raw_socket_stream.h
index bddd47eb75e1a9..23d557b1470010 100644
--- a/llvm/include/llvm/Support/raw_socket_stream.h
+++ b/llvm/include/llvm/Support/raw_socket_stream.h
@@ -110,6 +110,15 @@ class ListeningSocket {
static Expected<ListeningSocket> createUnix(
StringRef SocketPath,
int MaxBacklog = llvm::hardware_concurrency().compute_thread_count());
+
+ // The use of a default parameter was choosen over std::optional to more
+ // closely resemble the accept system call. A user should be able to call
+ // ListeningSocket.accept() rather then ListeningSocket.accept(std::nullopt)
+ // if they do not want accept to ever timeout
+ Expected<std::unique_ptr<raw_socket_stream>> accept(timeval TV = {-1, -1});
+ ListeningSocket(ListeningSocket &&LS);
+ void shutdown();
+ ~ListeningSocket();
};
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Support/raw_socket_stream.cpp b/llvm/lib/Support/raw_socket_stream.cpp
index 1dcf6352f2cc23..79521ca70d85f7 100644
--- a/llvm/lib/Support/raw_socket_stream.cpp
+++ b/llvm/lib/Support/raw_socket_stream.cpp
@@ -250,6 +250,7 @@ ListeningSocket::accept(std::chrono::milliseconds Timeout) {
}
void ListeningSocket::shutdown() {
+<<<<<<< HEAD
int ObservedFD = FD.load();
if (ObservedFD == -1)
@@ -259,6 +260,29 @@ void ListeningSocket::shutdown() {
// another thread is responsible for shutdown so return
if (!FD.compare_exchange_strong(ObservedFD, -1))
return;
+=======
+ if (FD == -1)
+ return;
+ ::close(FD);
+ ::unlink(SocketPath.c_str());
+
+ FD = -1;
+}
+
+ListeningSocket::~ListeningSocket() { shutdown(); }
+
+static Expected<int> GetSocketFD(StringRef SocketPath) {
+#ifdef _WIN32
+ SOCKET MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
+ if (MaybeWinsocket == INVALID_SOCKET) {
+#else
+ int MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
+ if (MaybeWinsocket == -1) {
+#endif // _WIN32
+ return llvm::make_error<StringError>(getLastSocketErrorCode(),
+ "Create socket failed");
+ }
+>>>>>>> [clang][MBD] set up module build daemon infrastructure
::close(ObservedFD);
::unlink(SocketPath.c_str());
>From 8ab80c27ca97056c1e313fb8e3ac229ed7efdde8 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Tue, 6 Feb 2024 16:48:44 -0500
Subject: [PATCH 02/20] Set ModuleBuildDaemonServer member functions to return
void
---
clang/tools/driver/cc1modbuildd_main.cpp | 17 +++++++----------
1 file changed, 7 insertions(+), 10 deletions(-)
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index e62bdd476abe83..234f8894bf6d19 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -60,9 +60,9 @@ class ModuleBuildDaemonServer {
~ModuleBuildDaemonServer() { shutdownDaemon(); }
- int setupDaemonEnv();
- int createDaemonSocket();
- int listenForClients();
+ void setupDaemonEnv();
+ void createDaemonSocket();
+ void listenForClients();
static void handleConnection(std::shared_ptr<raw_socket_stream> Connection);
@@ -89,7 +89,7 @@ void handleSignal(int) {
} // namespace
// Sets up file descriptors and signals for module build daemon
-int ModuleBuildDaemonServer::setupDaemonEnv() {
+void ModuleBuildDaemonServer::setupDaemonEnv() {
#ifdef _WIN32
freopen("NUL", "r", stdin);
@@ -117,12 +117,10 @@ int ModuleBuildDaemonServer::setupDaemonEnv() {
exit(EXIT_FAILURE);
}
#endif
-
- return EXIT_SUCCESS;
}
// Creates unix socket for IPC with frontends
-int ModuleBuildDaemonServer::createDaemonSocket() {
+void ModuleBuildDaemonServer::createDaemonSocket() {
Expected<ListeningSocket> MaybeServerListener =
llvm::ListeningSocket::createUnix(SocketPath);
@@ -149,7 +147,6 @@ int ModuleBuildDaemonServer::createDaemonSocket() {
verboseLog("mbd created and binded to socket at: " + SocketPath);
ServerListener.emplace(std::move(*MaybeServerListener));
- return 0;
}
// Function submitted to thread pool with each frontend connection. Not
@@ -178,7 +175,7 @@ void ModuleBuildDaemonServer::handleConnection(
return;
}
-int ModuleBuildDaemonServer::listenForClients() {
+void ModuleBuildDaemonServer::listenForClients() {
llvm::ThreadPool Pool;
@@ -200,10 +197,10 @@ int ModuleBuildDaemonServer::listenForClients() {
continue;
}
+ // Connection must be copy constructable to be passed to Pool.async
std::shared_ptr<raw_socket_stream> Connection(std::move(*MaybeConnection));
Pool.async(handleConnection, Connection);
}
- return 0;
}
// Module build daemon is spawned with the following command line:
>From 98bbe933c9437e93ffaef6d2e2a4c3bd046e2888 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Tue, 6 Feb 2024 17:35:09 -0500
Subject: [PATCH 03/20] Modify ListeningServer::accept to take
std::optional<std::chrono::microsecond>
---
.../clang/Tooling/ModuleBuildDaemon/Utils.h | 4 ++--
.../Tooling/ModuleBuildDaemon/Frontend.cpp | 9 ++++---
clang/tools/driver/cc1modbuildd_main.cpp | 4 +++-
llvm/include/llvm/Support/raw_socket_stream.h | 3 ++-
llvm/lib/Support/raw_socket_stream.cpp | 24 -------------------
5 files changed, 11 insertions(+), 33 deletions(-)
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
index 77fa4aee1e2ca9..c96a5a9540ad02 100644
--- a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
@@ -15,6 +15,7 @@
#include "llvm/Support/Error.h"
+#include <chrono>
#include <string>
#ifdef _WIN32
@@ -43,8 +44,7 @@ constexpr size_t SOCKET_ADDR_MAX_LENGTH = sizeof(sockaddr_un::sun_path);
constexpr size_t BASEPATH_MAX_LENGTH =
SOCKET_ADDR_MAX_LENGTH - SOCKET_FILE_NAME.length();
-// How long should the module build daemon sit ideal before exiting
-constexpr int TimeoutSec = 15;
+constexpr std::chrono::microseconds MICROSEC_IN_SEC(1000000);
// Get a temprary location where the daemon can store log files and a socket
// address. Of the format /tmp/clang-<BLAKE3HashOfClangFullVersion>/
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
index cf73e27f246bba..dad7d7f3802501 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -98,14 +98,13 @@ getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
spawnModuleBuildDaemon(Clang, Argv0, Diag, BasePath.str()))
return std::move(Err);
- constexpr unsigned int MICROSEC_IN_SEC = 1000000;
- constexpr unsigned int MAX_WAIT_TIME = 30 * MICROSEC_IN_SEC;
- unsigned int CumulativeTime = 0;
- unsigned int WaitTime = 10;
+ constexpr std::chrono::microseconds MAX_WAIT_TIME(30 * MICROSEC_IN_SEC);
+ std::chrono::microseconds CumulativeTime(0);
+ std::chrono::microseconds WaitTime(10);
while (CumulativeTime <= MAX_WAIT_TIME) {
// Wait a bit then check to see if the module build daemon has initialized
- std::this_thread::sleep_for(std::chrono::microseconds(WaitTime));
+ std::this_thread::sleep_for(WaitTime);
if (llvm::sys::fs::exists(SocketPath)) {
Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 234f8894bf6d19..4467a73c69a89d 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -178,10 +178,12 @@ void ModuleBuildDaemonServer::handleConnection(
void ModuleBuildDaemonServer::listenForClients() {
llvm::ThreadPool Pool;
+ // How long should the module build daemon sit ideal before exiting
+ std::chrono::microseconds DaemonTimeout(15 * MICROSEC_IN_SEC);
while (RunServiceLoop) {
Expected<std::unique_ptr<raw_socket_stream>> MaybeConnection =
- ServerListener.value().accept({TimeoutSec, 0});
+ ServerListener.value().accept(DaemonTimeout);
if (llvm::Error Err = MaybeConnection.takeError()) {
diff --git a/llvm/include/llvm/Support/raw_socket_stream.h b/llvm/include/llvm/Support/raw_socket_stream.h
index 23d557b1470010..2b54e20e959533 100644
--- a/llvm/include/llvm/Support/raw_socket_stream.h
+++ b/llvm/include/llvm/Support/raw_socket_stream.h
@@ -115,7 +115,8 @@ class ListeningSocket {
// closely resemble the accept system call. A user should be able to call
// ListeningSocket.accept() rather then ListeningSocket.accept(std::nullopt)
// if they do not want accept to ever timeout
- Expected<std::unique_ptr<raw_socket_stream>> accept(timeval TV = {-1, -1});
+ Expected<std::unique_ptr<raw_socket_stream>>
+ accept(std::optional<std::chrono::microseconds> Timeout = std::nullopt);
ListeningSocket(ListeningSocket &&LS);
void shutdown();
~ListeningSocket();
diff --git a/llvm/lib/Support/raw_socket_stream.cpp b/llvm/lib/Support/raw_socket_stream.cpp
index 79521ca70d85f7..1dcf6352f2cc23 100644
--- a/llvm/lib/Support/raw_socket_stream.cpp
+++ b/llvm/lib/Support/raw_socket_stream.cpp
@@ -250,7 +250,6 @@ ListeningSocket::accept(std::chrono::milliseconds Timeout) {
}
void ListeningSocket::shutdown() {
-<<<<<<< HEAD
int ObservedFD = FD.load();
if (ObservedFD == -1)
@@ -260,29 +259,6 @@ void ListeningSocket::shutdown() {
// another thread is responsible for shutdown so return
if (!FD.compare_exchange_strong(ObservedFD, -1))
return;
-=======
- if (FD == -1)
- return;
- ::close(FD);
- ::unlink(SocketPath.c_str());
-
- FD = -1;
-}
-
-ListeningSocket::~ListeningSocket() { shutdown(); }
-
-static Expected<int> GetSocketFD(StringRef SocketPath) {
-#ifdef _WIN32
- SOCKET MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
- if (MaybeWinsocket == INVALID_SOCKET) {
-#else
- int MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
- if (MaybeWinsocket == -1) {
-#endif // _WIN32
- return llvm::make_error<StringError>(getLastSocketErrorCode(),
- "Create socket failed");
- }
->>>>>>> [clang][MBD] set up module build daemon infrastructure
::close(ObservedFD);
::unlink(SocketPath.c_str());
>From 977c0247ce1452d87e898dcb30057a465487eab3 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Tue, 6 Feb 2024 21:46:02 -0500
Subject: [PATCH 04/20] Make raw_socket_stream FD atomic
---
llvm/include/llvm/Support/raw_socket_stream.h | 11 ++++-------
llvm/lib/Support/raw_socket_stream.cpp | 2 ++
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/Support/raw_socket_stream.h b/llvm/include/llvm/Support/raw_socket_stream.h
index 2b54e20e959533..7c912f376ab906 100644
--- a/llvm/include/llvm/Support/raw_socket_stream.h
+++ b/llvm/include/llvm/Support/raw_socket_stream.h
@@ -111,15 +111,12 @@ class ListeningSocket {
StringRef SocketPath,
int MaxBacklog = llvm::hardware_concurrency().compute_thread_count());
- // The use of a default parameter was choosen over std::optional to more
- // closely resemble the accept system call. A user should be able to call
- // ListeningSocket.accept() rather then ListeningSocket.accept(std::nullopt)
- // if they do not want accept to ever timeout
Expected<std::unique_ptr<raw_socket_stream>>
accept(std::optional<std::chrono::microseconds> Timeout = std::nullopt);
- ListeningSocket(ListeningSocket &&LS);
- void shutdown();
- ~ListeningSocket();
+
+ static Expected<ListeningSocket> createUnix(
+ StringRef SocketPath,
+ int MaxBacklog = llvm::hardware_concurrency().compute_thread_count());
};
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Support/raw_socket_stream.cpp b/llvm/lib/Support/raw_socket_stream.cpp
index 1dcf6352f2cc23..9b7ce02c170cda 100644
--- a/llvm/lib/Support/raw_socket_stream.cpp
+++ b/llvm/lib/Support/raw_socket_stream.cpp
@@ -20,6 +20,8 @@
#include <fcntl.h>
#include <thread>
+#include <atomic>
+
#ifndef _WIN32
#include <poll.h>
#include <sys/socket.h>
>From 17344af8a87339ecd9b145112185148f8291e7a6 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Tue, 6 Feb 2024 22:18:34 -0500
Subject: [PATCH 05/20] Remove exit in signal handler
---
clang/test/ModuleBuildDaemon/handshake.c | 6 ++++++
clang/tools/driver/cc1modbuildd_main.cpp | 14 +++++++-------
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/clang/test/ModuleBuildDaemon/handshake.c b/clang/test/ModuleBuildDaemon/handshake.c
index 059ecf6fae188c..84bdf9751131e7 100644
--- a/clang/test/ModuleBuildDaemon/handshake.c
+++ b/clang/test/ModuleBuildDaemon/handshake.c
@@ -20,3 +20,9 @@ int main() {return 0;}
// Check that a clang invocation can handshake with an existing module build daemon
// CHECK-EXIST: remark: Successfully completed handshake with module build daemon [-Rmodule-build-daemon]
+
+// Make sure mbd.sock does not exist
+// RUN: [ ! -f "mbd-launch/mbd.sock" ] && true || false
+
+// Make sure mbd.err is empty
+// RUN: [ ! -s "mbd-launch/mbd.err" ] && true || false
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 4467a73c69a89d..5253abc7dee94a 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -82,10 +82,7 @@ class ModuleBuildDaemonServer {
// Used to handle signals
ModuleBuildDaemonServer *DaemonPtr = nullptr;
-void handleSignal(int) {
- DaemonPtr->shutdownDaemon();
- exit(EXIT_SUCCESS);
-}
+void handleSignal(int) { DaemonPtr->shutdownDaemon(); }
} // namespace
// Sets up file descriptors and signals for module build daemon
@@ -178,7 +175,6 @@ void ModuleBuildDaemonServer::handleConnection(
void ModuleBuildDaemonServer::listenForClients() {
llvm::ThreadPool Pool;
- // How long should the module build daemon sit ideal before exiting
std::chrono::microseconds DaemonTimeout(15 * MICROSEC_IN_SEC);
while (RunServiceLoop) {
@@ -189,9 +185,13 @@ void ModuleBuildDaemonServer::listenForClients() {
llvm::handleAllErrors(std::move(Err), [&](const llvm::StringError &SE) {
std::error_code EC = SE.convertToErrorCode();
- if (EC == std::errc::timed_out)
+
+ if (EC == std::errc::timed_out) {
RunServiceLoop = false;
- else
+ verboseLog("ListeningServer::accept timed out, shutting down");
+ } else if (EC == std::errc::interrupted && RunServiceLoop == false) {
+ verboseLog("Signal received, shutting down");
+ } else
errs() << "MBD failed to accept incoming connection: "
<< SE.getMessage() << ": " << EC.message() << '\n';
});
>From bd3166c3808a3ebc3bbe51b4e87a43a33d2544d0 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Wed, 7 Feb 2024 01:24:12 -0500
Subject: [PATCH 06/20] Create setupSignal functionality so I can set signals
back to their default state after ServerListener is destructed
---
clang/tools/driver/cc1modbuildd_main.cpp | 72 +++++++++++++++---------
1 file changed, 46 insertions(+), 26 deletions(-)
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 5253abc7dee94a..12475d98ccbd0a 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -43,6 +43,34 @@ static void verboseLog(const llvm::Twine &message) {
}
}
+static void setupSignals(sighandler_t handler) {
+
+ if (std::signal(SIGTERM, handler) == SIG_ERR) {
+ errs() << "failed to handle SIGTERM" << '\n';
+ exit(EXIT_FAILURE);
+ }
+
+ if (std::signal(SIGINT, handler) == SIG_ERR) {
+ errs() << "failed to handle SIGINT" << '\n';
+ exit(EXIT_FAILURE);
+ }
+
+// TODO: Figure out how to do this on windows
+#ifdef SIGHUP
+ if (handler != SIG_DFL) {
+ if (::signal(SIGHUP, SIG_IGN) == SIG_ERR) {
+ errs() << "failed to handle SIGHUP" << '\n';
+ exit(EXIT_FAILURE);
+ }
+ } else {
+ if (::signal(SIGHUP, SIG_DFL) == SIG_ERR) {
+ errs() << "failed to handle SIGHUP" << '\n';
+ exit(EXIT_FAILURE);
+ }
+ }
+#endif
+}
+
namespace {
class ModuleBuildDaemonServer {
@@ -58,8 +86,6 @@ class ModuleBuildDaemonServer {
llvm::sys::path::append(STDERR, STDERR_FILE_NAME);
}
- ~ModuleBuildDaemonServer() { shutdownDaemon(); }
-
void setupDaemonEnv();
void createDaemonSocket();
void listenForClients();
@@ -73,6 +99,10 @@ class ModuleBuildDaemonServer {
RunServiceLoop = false;
if (ServerListener.has_value())
ServerListener.value().shutdown();
+
+ // Prevents the ServerListener destructor being called after
+ // ServerListener::shutdown is run. We do not want to call ::unlink twice
+ exit(0);
}
private:
@@ -97,23 +127,7 @@ void ModuleBuildDaemonServer::setupDaemonEnv() {
freopen(STDOUT.c_str(), "a", stdout);
freopen(STDERR.c_str(), "a", stderr);
- if (std::signal(SIGTERM, handleSignal) == SIG_ERR) {
- errs() << "failed to handle SIGTERM" << '\n';
- exit(EXIT_FAILURE);
- }
-
- if (std::signal(SIGINT, handleSignal) == SIG_ERR) {
- errs() << "failed to handle SIGINT" << '\n';
- exit(EXIT_FAILURE);
- }
-
-// TODO: Figure out how to do this on windows
-#ifdef SIGHUP
- if (::signal(SIGHUP, SIG_IGN) == SIG_ERR) {
- errs() << "failed to handle SIGHUP" << '\n';
- exit(EXIT_FAILURE);
- }
-#endif
+ setupSignals(handleSignal);
}
// Creates unix socket for IPC with frontends
@@ -237,18 +251,24 @@ int cc1modbuildd_main(ArrayRef<const char *> Argv) {
if (!validBasePathLength(BasePath)) {
errs() << "BasePath '" << BasePath << "' is longer then the max length of "
<< std::to_string(BASEPATH_MAX_LENGTH) << '\n';
- return 1;
+ return EXIT_FAILURE;
}
llvm::sys::fs::create_directories(BasePath);
- ModuleBuildDaemonServer Daemon(BasePath);
- // Used to handle signals
- DaemonPtr = &Daemon;
+ {
+ ModuleBuildDaemonServer Daemon(BasePath);
+
+ // Used to handle signals
+ DaemonPtr = &Daemon;
+
+ Daemon.setupDaemonEnv();
+ Daemon.createDaemonSocket();
+ Daemon.listenForClients();
+ }
- Daemon.setupDaemonEnv();
- Daemon.createDaemonSocket();
- Daemon.listenForClients();
+ // Prevents handleSignal from being called after ServerListener is destructed
+ setupSignals(SIG_DFL);
return EXIT_SUCCESS;
}
>From 60c8d0aaea09492442c4db26e5ec733eb39617bf Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Wed, 7 Feb 2024 01:24:50 -0500
Subject: [PATCH 07/20] Make blacker happy
---
clang/utils/kill_process.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/clang/utils/kill_process.py b/clang/utils/kill_process.py
index 68d0e4ed8dc629..cf277e43b9d097 100644
--- a/clang/utils/kill_process.py
+++ b/clang/utils/kill_process.py
@@ -18,7 +18,6 @@ def windows_impl(search_pattern):
for line in lines:
if "clang" in line and "kill_process" not in line and "llvm-lit" not in line:
-
# creates something like ['..\\llvm-project\\build\\bin\\clang.exe -cc1modbuildd mbd-launch -v', '16260']
parts = line.rsplit(None, 1)
>From aace6b5d9ac1939add633ea1f3c46a531720bb39 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Wed, 7 Feb 2024 03:02:07 -0500
Subject: [PATCH 08/20] Fix bug where if daemon crashes and does not unlink
socket file the next daemon can remove the file before trying to bind
---
.../Tooling/ModuleBuildDaemon/Frontend.cpp | 3 +-
clang/tools/driver/cc1modbuildd_main.cpp | 51 +++++++++++--------
2 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
index dad7d7f3802501..8a456ff728fb40 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -88,9 +88,8 @@ getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
if (llvm::sys::fs::exists(SocketPath)) {
Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
connectToSocket(SocketPath);
- if (MaybeClient) {
+ if (MaybeClient)
return std::move(*MaybeClient);
- }
consumeError(MaybeClient.takeError());
}
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 12475d98ccbd0a..4bcee57536a9dd 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -133,31 +133,42 @@ void ModuleBuildDaemonServer::setupDaemonEnv() {
// Creates unix socket for IPC with frontends
void ModuleBuildDaemonServer::createDaemonSocket() {
- Expected<ListeningSocket> MaybeServerListener =
- llvm::ListeningSocket::createUnix(SocketPath);
-
- if (llvm::Error Err = MaybeServerListener.takeError()) {
- llvm::handleAllErrors(std::move(Err), [&](const llvm::StringError &SE) {
- std::error_code EC = SE.convertToErrorCode();
- // Exit successfully if the socket address is already in use. When
- // translation units are compiled in parallel, until the socket file is
- // created, all clang invocations will try to spawn a module build daemon.
+ bool SocketCreated = false;
+ while (!SocketCreated) {
+
+ Expected<ListeningSocket> MaybeServerListener =
+ llvm::ListeningSocket::createUnix(SocketPath);
+
+ if (llvm::Error Err = MaybeServerListener.takeError()) {
+ llvm::handleAllErrors(std::move(Err), [&](const llvm::StringError &SE) {
+ std::error_code EC = SE.convertToErrorCode();
+
+ // Exit successfully if the socket address is already in use. When
+ // TUs are compiled in parallel, until the socket file is created, all
+ // clang invocations will try to spawn a module build daemon.
#ifdef _WIN32
- if (EC.value() == WSAEADDRINUSE) {
+ if (EC.value() == WSAEADDRINUSE) {
#else
if (EC == std::errc::address_in_use) {
#endif
- exit(EXIT_SUCCESS);
- } else {
- llvm::errs() << "MBD failed to create unix socket: " << SE.message()
- << EC.message() << '\n';
- exit(EXIT_FAILURE);
- }
- });
+ exit(EXIT_SUCCESS);
+ } else if (EC == std::errc::file_exists) {
+ if (std::error_code EC = llvm::sys::fs::remove(SocketPath); EC) {
+ llvm::errs() << "Failed to remove file: " << EC.message() << '\n';
+ exit(EXIT_FAILURE);
+ }
+ } else {
+ llvm::errs() << "MBD failed to create unix socket: "
+ << SE.getMessage() << ": " << EC.message() << '\n';
+ exit(EXIT_FAILURE);
+ }
+ });
+ } else {
+ SocketCreated = true;
+ verboseLog("mbd created and binded to socket at: " + SocketPath);
+ ServerListener.emplace(std::move(*MaybeServerListener));
+ }
}
-
- verboseLog("mbd created and binded to socket at: " + SocketPath);
- ServerListener.emplace(std::move(*MaybeServerListener));
}
// Function submitted to thread pool with each frontend connection. Not
>From 0940e63e345083778244c4ad1d9961717f520670 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sat, 10 Feb 2024 16:19:21 -0500
Subject: [PATCH 09/20] Remove std::move(s) preventing copy elision
-Wpessimizing-move
---
clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h | 2 +-
clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp | 4 ++--
clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h b/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
index 94a10359c61471..88c71be936f5af 100644
--- a/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
@@ -94,7 +94,7 @@ llvm::Error writeMsgStructToSocket(llvm::raw_socket_stream &Socket,
std::string Buffer = convertMsgStructToBuffer(MsgStruct);
if (llvm::Error Err = writeBufferToSocket(Socket, Buffer))
- return std::move(Err);
+ return Err;
return llvm::Error::success();
}
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
index 8a456ff728fb40..e7fffa9823568f 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -35,13 +35,13 @@ llvm::Error attemptHandshake(raw_socket_stream &Client,
// Send HandshakeMsg to module build daemon
HandshakeMsg Request{ActionType::HANDSHAKE, StatusType::REQUEST};
if (llvm::Error Err = writeMsgStructToSocket(Client, Request))
- return std::move(Err);
+ return Err;
// Read response from module build daemon
Expected<HandshakeMsg> MaybeResponse =
readMsgStructFromSocket<HandshakeMsg>(Client);
if (!MaybeResponse) {
- return std::move(MaybeResponse.takeError());
+ return MaybeResponse.takeError();
}
HandshakeMsg Response = std::move(*MaybeResponse);
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp b/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
index 08ddfff8c7f387..be0665147a0a45 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
@@ -23,7 +23,7 @@ connectToSocket(StringRef SocketPath) {
Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
raw_socket_stream::createConnectedUnix(SocketPath);
if (!MaybeClient)
- return std::move(MaybeClient.takeError());
+ return MaybeClient.takeError();
return std::move(*MaybeClient);
}
>From 88bbaf8508fed84b5db9848824f9e0556c19f165 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sat, 10 Feb 2024 21:08:01 -0500
Subject: [PATCH 10/20] Remove exit in shutdownDeamon because ListeningSocket
destructor will return if FD == -1
---
clang/tools/driver/cc1modbuildd_main.cpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 4bcee57536a9dd..03f6b78590f595 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -99,10 +99,6 @@ class ModuleBuildDaemonServer {
RunServiceLoop = false;
if (ServerListener.has_value())
ServerListener.value().shutdown();
-
- // Prevents the ServerListener destructor being called after
- // ServerListener::shutdown is run. We do not want to call ::unlink twice
- exit(0);
}
private:
>From f8cd6ed2b395fb8ab7fb7df7ca367c50bc169063 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sat, 10 Feb 2024 21:18:28 -0500
Subject: [PATCH 11/20] Take advantage of ability to convert between
std::seconds and std::microseconds
---
clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h | 2 --
clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp | 4 ++--
clang/tools/driver/cc1modbuildd_main.cpp | 2 +-
3 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
index c96a5a9540ad02..b37eb18210b344 100644
--- a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
@@ -44,8 +44,6 @@ constexpr size_t SOCKET_ADDR_MAX_LENGTH = sizeof(sockaddr_un::sun_path);
constexpr size_t BASEPATH_MAX_LENGTH =
SOCKET_ADDR_MAX_LENGTH - SOCKET_FILE_NAME.length();
-constexpr std::chrono::microseconds MICROSEC_IN_SEC(1000000);
-
// Get a temprary location where the daemon can store log files and a socket
// address. Of the format /tmp/clang-<BLAKE3HashOfClangFullVersion>/
std::string getBasePath();
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
index e7fffa9823568f..3b2688417cbc70 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -97,11 +97,11 @@ getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
spawnModuleBuildDaemon(Clang, Argv0, Diag, BasePath.str()))
return std::move(Err);
- constexpr std::chrono::microseconds MAX_WAIT_TIME(30 * MICROSEC_IN_SEC);
+ std::chrono::seconds MaxWaitTime(30);
std::chrono::microseconds CumulativeTime(0);
std::chrono::microseconds WaitTime(10);
- while (CumulativeTime <= MAX_WAIT_TIME) {
+ while (CumulativeTime <= MaxWaitTime) {
// Wait a bit then check to see if the module build daemon has initialized
std::this_thread::sleep_for(WaitTime);
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 03f6b78590f595..6f1d0d4d3630ec 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -196,7 +196,7 @@ void ModuleBuildDaemonServer::handleConnection(
void ModuleBuildDaemonServer::listenForClients() {
llvm::ThreadPool Pool;
- std::chrono::microseconds DaemonTimeout(15 * MICROSEC_IN_SEC);
+ std::chrono::seconds DaemonTimeout(15);
while (RunServiceLoop) {
Expected<std::unique_ptr<raw_socket_stream>> MaybeConnection =
>From 6bb9cbee2a8e9398516d048ad642e4b76730c929 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sat, 10 Feb 2024 21:22:43 -0500
Subject: [PATCH 12/20] Switch to break statement to get rid of state whn
calling createUnix
---
clang/tools/driver/cc1modbuildd_main.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 6f1d0d4d3630ec..a52c2294e75a69 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -129,9 +129,7 @@ void ModuleBuildDaemonServer::setupDaemonEnv() {
// Creates unix socket for IPC with frontends
void ModuleBuildDaemonServer::createDaemonSocket() {
- bool SocketCreated = false;
- while (!SocketCreated) {
-
+ while (true) {
Expected<ListeningSocket> MaybeServerListener =
llvm::ListeningSocket::createUnix(SocketPath);
@@ -160,9 +158,9 @@ void ModuleBuildDaemonServer::createDaemonSocket() {
}
});
} else {
- SocketCreated = true;
verboseLog("mbd created and binded to socket at: " + SocketPath);
ServerListener.emplace(std::move(*MaybeServerListener));
+ break;
}
}
}
>From c78a0a824ca159a94b46db7ebf249a3e5c2c6b04 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sat, 10 Feb 2024 21:35:01 -0500
Subject: [PATCH 13/20] Fix the style of a few variable names
---
.../clang/Tooling/ModuleBuildDaemon/Utils.h | 14 +++++++-------
.../lib/Tooling/ModuleBuildDaemon/Frontend.cpp | 6 +++---
clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp | 4 ++--
clang/tools/driver/cc1modbuildd_main.cpp | 18 +++++++++---------
4 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
index b37eb18210b344..df721cc8a91861 100644
--- a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
@@ -33,16 +33,16 @@
namespace clang::tooling::cc1modbuildd {
-constexpr std::string_view SOCKET_FILE_NAME = "mbd.sock";
-constexpr std::string_view STDOUT_FILE_NAME = "mbd.out";
-constexpr std::string_view STDERR_FILE_NAME = "mbd.err";
-constexpr std::string_view MODULE_BUILD_DAEMON_FLAG = "-cc1modbuildd";
+constexpr std::string_view SocketFileName = "mbd.sock";
+constexpr std::string_view StdoutFileName = "mbd.out";
+constexpr std::string_view StderrFileName = "mbd.err";
+constexpr std::string_view ModuleBuildDaemonFlag = "-cc1modbuildd";
// A llvm::raw_socket_stream uses sockaddr_un
-constexpr size_t SOCKET_ADDR_MAX_LENGTH = sizeof(sockaddr_un::sun_path);
+constexpr size_t SocketAddrMaxLength = sizeof(sockaddr_un::sun_path);
-constexpr size_t BASEPATH_MAX_LENGTH =
- SOCKET_ADDR_MAX_LENGTH - SOCKET_FILE_NAME.length();
+constexpr size_t BasePathMaxLength =
+ SocketAddrMaxLength - SocketFileName.length();
// Get a temprary location where the daemon can store log files and a socket
// address. Of the format /tmp/clang-<BLAKE3HashOfClangFullVersion>/
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
index 3b2688417cbc70..8c7f440ad79236 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -61,7 +61,7 @@ llvm::Error spawnModuleBuildDaemon(const CompilerInvocation &Clang,
const char *Argv0, DiagnosticsEngine &Diag,
std::string BasePath) {
- std::vector<StringRef> Args = {Argv0, MODULE_BUILD_DAEMON_FLAG};
+ std::vector<StringRef> Args = {Argv0, ModuleBuildDaemonFlag};
if (!Clang.getFrontendOpts().ModuleBuildDaemonPath.empty())
Args.push_back(BasePath.c_str());
@@ -83,7 +83,7 @@ getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
DiagnosticsEngine &Diag, StringRef BasePath) {
SmallString<128> SocketPath = BasePath;
- llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+ llvm::sys::path::append(SocketPath, SocketFileName);
if (llvm::sys::fs::exists(SocketPath)) {
Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
@@ -139,7 +139,7 @@ void spawnModuleBuildDaemonAndHandshake(const CompilerInvocation &Clang,
// Get user provided BasePath and confirm it is short enough
BasePath = Clang.getFrontendOpts().ModuleBuildDaemonPath;
if (!validBasePathLength(BasePath)) {
- Diag.Report(diag::err_basepath_length) << BasePath << BASEPATH_MAX_LENGTH;
+ Diag.Report(diag::err_basepath_length) << BasePath << BasePathMaxLength;
return;
}
}
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
index 93fc14c4ba23dc..9ce1ba849bd83c 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
@@ -39,8 +39,8 @@ std::string getBasePath() {
bool validBasePathLength(llvm::StringRef Address) {
// Confirm that the user provided BasePath is short enough to allow the socket
- // address to fit within the alloted space
- if (Address.str().length() > BASEPATH_MAX_LENGTH) {
+ // address to fit within the space alloted to sockaddr_un::sun_path
+ if (Address.str().length() > BasePathMaxLength) {
return false;
}
return true;
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index a52c2294e75a69..49a62006cdcec3 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -76,14 +76,14 @@ namespace {
class ModuleBuildDaemonServer {
public:
SmallString<256> SocketPath;
- SmallString<256> STDERR;
- SmallString<256> STDOUT;
+ SmallString<256> Stderr; // path to stderr
+ SmallString<256> Stdout; // path to stdout
ModuleBuildDaemonServer(StringRef Path)
- : SocketPath(Path), STDERR(Path), STDOUT(Path) {
- llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
- llvm::sys::path::append(STDOUT, STDOUT_FILE_NAME);
- llvm::sys::path::append(STDERR, STDERR_FILE_NAME);
+ : SocketPath(Path), Stderr(Path), Stdout(Path) {
+ llvm::sys::path::append(SocketPath, SocketFileName);
+ llvm::sys::path::append(Stdout, StdoutFileName);
+ llvm::sys::path::append(Stderr, StderrFileName);
}
void setupDaemonEnv();
@@ -120,8 +120,8 @@ void ModuleBuildDaemonServer::setupDaemonEnv() {
close(STDIN_FILENO);
#endif
- freopen(STDOUT.c_str(), "a", stdout);
- freopen(STDERR.c_str(), "a", stderr);
+ freopen(Stdout.c_str(), "a", stdout);
+ freopen(Stderr.c_str(), "a", stderr);
setupSignals(handleSignal);
}
@@ -255,7 +255,7 @@ int cc1modbuildd_main(ArrayRef<const char *> Argv) {
if (!validBasePathLength(BasePath)) {
errs() << "BasePath '" << BasePath << "' is longer then the max length of "
- << std::to_string(BASEPATH_MAX_LENGTH) << '\n';
+ << std::to_string(BasePathMaxLength) << '\n';
return EXIT_FAILURE;
}
>From 658de792abfa473a5edb605e4846a32eaac07703 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sat, 10 Feb 2024 21:41:18 -0500
Subject: [PATCH 14/20] After ModuleBuildDaemonServer deconstructs ignore
signals
---
clang/tools/driver/cc1modbuildd_main.cpp | 21 +++++++--------------
1 file changed, 7 insertions(+), 14 deletions(-)
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 49a62006cdcec3..32841df1f58e7e 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -43,7 +43,7 @@ static void verboseLog(const llvm::Twine &message) {
}
}
-static void setupSignals(sighandler_t handler) {
+static void modifySignals(sighandler_t handler) {
if (std::signal(SIGTERM, handler) == SIG_ERR) {
errs() << "failed to handle SIGTERM" << '\n';
@@ -55,18 +55,10 @@ static void setupSignals(sighandler_t handler) {
exit(EXIT_FAILURE);
}
-// TODO: Figure out how to do this on windows
#ifdef SIGHUP
- if (handler != SIG_DFL) {
- if (::signal(SIGHUP, SIG_IGN) == SIG_ERR) {
- errs() << "failed to handle SIGHUP" << '\n';
- exit(EXIT_FAILURE);
- }
- } else {
- if (::signal(SIGHUP, SIG_DFL) == SIG_ERR) {
- errs() << "failed to handle SIGHUP" << '\n';
- exit(EXIT_FAILURE);
- }
+ if (::signal(SIGHUP, SIG_IGN) == SIG_ERR) {
+ errs() << "failed to handle SIGHUP" << '\n';
+ exit(EXIT_FAILURE);
}
#endif
}
@@ -123,7 +115,7 @@ void ModuleBuildDaemonServer::setupDaemonEnv() {
freopen(Stdout.c_str(), "a", stdout);
freopen(Stderr.c_str(), "a", stderr);
- setupSignals(handleSignal);
+ modifySignals(handleSignal);
}
// Creates unix socket for IPC with frontends
@@ -273,7 +265,8 @@ int cc1modbuildd_main(ArrayRef<const char *> Argv) {
}
// Prevents handleSignal from being called after ServerListener is destructed
- setupSignals(SIG_DFL);
+ // Daemon is shutting down at this point so signals can be ignored
+ modifySignals(SIG_IGN);
return EXIT_SUCCESS;
}
>From aa332f5be099d74816f108e8416b0aa76c2e56d7 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sun, 11 Feb 2024 00:29:17 -0500
Subject: [PATCH 15/20] Switch over to using non-deprecated version of close
and unlink for windows and improve remove socket file capabilities
---
clang/tools/driver/cc1modbuildd_main.cpp | 7 +++++--
llvm/include/llvm/Support/raw_socket_stream.h | 1 +
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 32841df1f58e7e..e2ffc06771ed52 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -15,6 +15,8 @@
#include <csignal>
#include <cstdbool>
+#include <cstdio>
+#include <cstring>
#include <fstream>
#include <optional>
#include <string>
@@ -139,8 +141,9 @@ void ModuleBuildDaemonServer::createDaemonSocket() {
#endif
exit(EXIT_SUCCESS);
} else if (EC == std::errc::file_exists) {
- if (std::error_code EC = llvm::sys::fs::remove(SocketPath); EC) {
- llvm::errs() << "Failed to remove file: " << EC.message() << '\n';
+ if (std::remove(SocketPath.c_str()) != 0) {
+ llvm::errs() << "Failed to remove file: " << strerror(errno)
+ << '\n';
exit(EXIT_FAILURE);
}
} else {
diff --git a/llvm/include/llvm/Support/raw_socket_stream.h b/llvm/include/llvm/Support/raw_socket_stream.h
index 7c912f376ab906..0c966633e38f9c 100644
--- a/llvm/include/llvm/Support/raw_socket_stream.h
+++ b/llvm/include/llvm/Support/raw_socket_stream.h
@@ -130,6 +130,7 @@ class raw_socket_stream : public raw_fd_stream {
#endif // _WIN32
public:
+ // TODO: Should probably be private
raw_socket_stream(int SocketFD);
/// Create a \p raw_socket_stream connected to the UNIX domain socket at \p
/// SocketPath.
>From a0bfafa5f5a4d61d4554c80dec840e16c0e93c41 Mon Sep 17 00:00:00 2001
From: Connor Sughrue <cpsughrue at gmail.com>
Date: Mon, 12 Feb 2024 23:01:09 -0500
Subject: [PATCH 16/20] fix windows build and formatting issues
---
clang/tools/driver/cc1modbuildd_main.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index e2ffc06771ed52..3764f4909bc2be 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -45,7 +45,7 @@ static void verboseLog(const llvm::Twine &message) {
}
}
-static void modifySignals(sighandler_t handler) {
+static void modifySignals(decltype(SIG_DFL) handler) {
if (std::signal(SIGTERM, handler) == SIG_ERR) {
errs() << "failed to handle SIGTERM" << '\n';
>From e087788d438e4c04fc0cb53538137be3cc306e47 Mon Sep 17 00:00:00 2001
From: Connor Sughrue <cpsughrue at gmail.com>
Date: Tue, 13 Feb 2024 20:37:15 -0500
Subject: [PATCH 17/20] Remove check that mbd.sock was removed - fails on
windows due to how files are deleted
---
clang/test/ModuleBuildDaemon/daemon-launch.c | 3 ---
clang/test/ModuleBuildDaemon/handshake.c | 3 ---
2 files changed, 6 deletions(-)
diff --git a/clang/test/ModuleBuildDaemon/daemon-launch.c b/clang/test/ModuleBuildDaemon/daemon-launch.c
index 2bfea6aec9dccd..00e94e9ba456df 100644
--- a/clang/test/ModuleBuildDaemon/daemon-launch.c
+++ b/clang/test/ModuleBuildDaemon/daemon-launch.c
@@ -11,8 +11,5 @@
// CHECK: mbd created and binded to socket at: mbd-launch/mbd.sock
-// Make sure mbd.sock does not exist
-// RUN: [ ! -f "mbd-launch/mbd.sock" ] && true || false
-
// Make sure mbd.err is empty
// RUN: [ ! -s "mbd-launch/mbd.err" ] && true || false
diff --git a/clang/test/ModuleBuildDaemon/handshake.c b/clang/test/ModuleBuildDaemon/handshake.c
index 84bdf9751131e7..af71daa89e90db 100644
--- a/clang/test/ModuleBuildDaemon/handshake.c
+++ b/clang/test/ModuleBuildDaemon/handshake.c
@@ -21,8 +21,5 @@ int main() {return 0;}
// Check that a clang invocation can handshake with an existing module build daemon
// CHECK-EXIST: remark: Successfully completed handshake with module build daemon [-Rmodule-build-daemon]
-// Make sure mbd.sock does not exist
-// RUN: [ ! -f "mbd-launch/mbd.sock" ] && true || false
-
// Make sure mbd.err is empty
// RUN: [ ! -s "mbd-launch/mbd.err" ] && true || false
>From 28a8bb37e46cdf0f48824e8814fc3259785c5017 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Tue, 13 Feb 2024 21:52:05 -0500
Subject: [PATCH 18/20] Write test to crash situation where MBD crashes
---
clang/test/ModuleBuildDaemon/daemon-crash.c | 20 ++++++++++++++++++++
clang/test/ModuleBuildDaemon/daemon-launch.c | 3 ++-
clang/tools/driver/cc1modbuildd_main.cpp | 11 +++++++----
3 files changed, 29 insertions(+), 5 deletions(-)
create mode 100644 clang/test/ModuleBuildDaemon/daemon-crash.c
diff --git a/clang/test/ModuleBuildDaemon/daemon-crash.c b/clang/test/ModuleBuildDaemon/daemon-crash.c
new file mode 100644
index 00000000000000..67bccfc3489b5c
--- /dev/null
+++ b/clang/test/ModuleBuildDaemon/daemon-crash.c
@@ -0,0 +1,20 @@
+// Check that the module build daemon can create a unix socket after a crash
+
+// RUN: rm -rf mbd-crash %t
+
+// timeout should exit with status 124 which is treated as a failure by lit on
+// windows. Ideally we would be like to check the exit code and only return true
+// if it equals 124 but lit does not support global bash sysmbols like $?
+
+// RUN: timeout --signal=SIGKILL 2 %clang -cc1modbuildd mbd-crash -v || true
+// RUN: timeout --signal=SIGTERM 2 %clang -cc1modbuildd mbd-crash -v || true
+// RUN: cat mbd-crash/mbd.out | sed 's:\\\\\?:/:g' | FileCheck %s
+
+// There should only be one shutdown log line due to the crash
+// CHECK: MBD created and binded to socket at: mbd-crash/mbd.sock
+// CHECK-NEXT: Removing ineligible file: mbd-crash/mbd.sock
+// CHECK-NEXT: MBD created and binded to socket at: mbd-crash/mbd.sock
+// CHECK-NEXT: Signal received, shutting down
+
+// Make sure mbd.err is empty
+// RUN: [ ! -s "mbd-launch/mbd.err" ] && true || false
diff --git a/clang/test/ModuleBuildDaemon/daemon-launch.c b/clang/test/ModuleBuildDaemon/daemon-launch.c
index 00e94e9ba456df..9c148ae029cf3c 100644
--- a/clang/test/ModuleBuildDaemon/daemon-launch.c
+++ b/clang/test/ModuleBuildDaemon/daemon-launch.c
@@ -9,7 +9,8 @@
// RUN: timeout --signal=SIGTERM 2 %clang -cc1modbuildd mbd-launch -v || true
// RUN: cat mbd-launch/mbd.out | sed 's:\\\\\?:/:g' | FileCheck %s
-// CHECK: mbd created and binded to socket at: mbd-launch/mbd.sock
+// CHECK: MBD created and binded to socket at: mbd-launch/mbd.sock
+// CHECK-NEXT: Signal received, shutting down
// Make sure mbd.err is empty
// RUN: [ ! -s "mbd-launch/mbd.err" ] && true || false
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index 3764f4909bc2be..f5c6e4e0ca0791 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -137,15 +137,18 @@ void ModuleBuildDaemonServer::createDaemonSocket() {
#ifdef _WIN32
if (EC.value() == WSAEADDRINUSE) {
#else
- if (EC == std::errc::address_in_use) {
+ if (EC == std::errc::address_in_use) {
#endif
exit(EXIT_SUCCESS);
} else if (EC == std::errc::file_exists) {
if (std::remove(SocketPath.c_str()) != 0) {
- llvm::errs() << "Failed to remove file: " << strerror(errno)
- << '\n';
+ llvm::errs() << "Failed to remove " << SocketPath << ": "
+ << strerror(errno) << '\n';
exit(EXIT_FAILURE);
}
+ // If a previous module build daemon invocation crashes, the socket
+ // file will need to be removed before the address can be binded to
+ verboseLog("Removing ineligible file: " + SocketPath);
} else {
llvm::errs() << "MBD failed to create unix socket: "
<< SE.getMessage() << ": " << EC.message() << '\n';
@@ -153,7 +156,7 @@ void ModuleBuildDaemonServer::createDaemonSocket() {
}
});
} else {
- verboseLog("mbd created and binded to socket at: " + SocketPath);
+ verboseLog("MBD created and binded to socket at: " + SocketPath);
ServerListener.emplace(std::move(*MaybeServerListener));
break;
}
>From a3a5ad192384125aec0564638c752ec0d6a77556 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Wed, 10 Apr 2024 14:07:10 -0400
Subject: [PATCH 19/20] Build fixes after rebase
---
clang/include/clang/Frontend/FrontendOptions.h | 2 +-
clang/tools/driver/cc1modbuildd_main.cpp | 2 +-
llvm/include/llvm/Support/raw_socket_stream.h | 7 -------
3 files changed, 2 insertions(+), 9 deletions(-)
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 7ea87acbecc230..1bde079dc575e7 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -403,7 +403,7 @@ class FrontendOptions {
/// Whether to emit symbol labels for testing in generated symbol graphs
LLVM_PREFERRED_TYPE(bool)
unsigned EmitPrettySymbolGraphs : 1;
-
+
/// Connect to module build daemon.
LLVM_PREFERRED_TYPE(bool)
unsigned ModuleBuildDaemon : 1;
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
index f5c6e4e0ca0791..a99ecb8a3dc1c0 100644
--- a/clang/tools/driver/cc1modbuildd_main.cpp
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -191,7 +191,7 @@ void ModuleBuildDaemonServer::handleConnection(
void ModuleBuildDaemonServer::listenForClients() {
- llvm::ThreadPool Pool;
+ llvm::DefaultThreadPool Pool;
std::chrono::seconds DaemonTimeout(15);
while (RunServiceLoop) {
diff --git a/llvm/include/llvm/Support/raw_socket_stream.h b/llvm/include/llvm/Support/raw_socket_stream.h
index 0c966633e38f9c..38c471893624dd 100644
--- a/llvm/include/llvm/Support/raw_socket_stream.h
+++ b/llvm/include/llvm/Support/raw_socket_stream.h
@@ -110,13 +110,6 @@ class ListeningSocket {
static Expected<ListeningSocket> createUnix(
StringRef SocketPath,
int MaxBacklog = llvm::hardware_concurrency().compute_thread_count());
-
- Expected<std::unique_ptr<raw_socket_stream>>
- accept(std::optional<std::chrono::microseconds> Timeout = std::nullopt);
-
- static Expected<ListeningSocket> createUnix(
- StringRef SocketPath,
- int MaxBacklog = llvm::hardware_concurrency().compute_thread_count());
};
//===----------------------------------------------------------------------===//
>From d04f4afa7045911d82b4ed80b2d2517eb36c9832 Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Wed, 10 Apr 2024 14:16:48 -0400
Subject: [PATCH 20/20] Use llvm support ExponentialBackoff function
---
clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
index 8c7f440ad79236..40ec386784992a 100644
--- a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -15,6 +15,7 @@
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ExponentialBackoff.h"
#include "llvm/Support/Program.h"
#include <cerrno>
@@ -98,13 +99,8 @@ getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
return std::move(Err);
std::chrono::seconds MaxWaitTime(30);
- std::chrono::microseconds CumulativeTime(0);
- std::chrono::microseconds WaitTime(10);
-
- while (CumulativeTime <= MaxWaitTime) {
- // Wait a bit then check to see if the module build daemon has initialized
- std::this_thread::sleep_for(WaitTime);
-
+ ExponentialBackoff Backoff(MaxWaitTime);
+ do {
if (llvm::sys::fs::exists(SocketPath)) {
Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
connectToSocket(SocketPath);
@@ -114,10 +110,7 @@ getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
}
consumeError(MaybeClient.takeError());
}
-
- CumulativeTime += WaitTime;
- WaitTime = WaitTime * 2;
- }
+ } while (Backoff.waitForNextAttempt());
// After waiting around 30 seconds give up and return an error
return llvm::make_error<StringError>(
More information about the llvm-commits
mailing list