[clang] [llvm] [clang][MBD] set up module build daemon infrastructure (PR #67562)
Connor Sughrue via cfe-commits
cfe-commits at lists.llvm.org
Tue Feb 6 13:07:04 PST 2024
https://github.com/cpsughrue updated https://github.com/llvm/llvm-project/pull/67562
>From 823e05ca073366f54996a74b2a6d6661ed4e575a Mon Sep 17 00:00:00 2001
From: cpsughrue <cpsughrue at gmail.com>
Date: Sun, 9 Jul 2023 23:19:58 -0400
Subject: [PATCH] [clang][MBD] set up module build daemon infrastructure
The module build daemon (mbd) will serve as a cc1 tool that helps convert
implicit module command lines to explicit module command lines. A clang
invocation will check to see if a mbd exists, if not the invocation will spawn
one. After the mbd is up and running and a handshake has successfully been
carried out between the mbd and clang invocation the clang invocation will
share it's command line with the mbd. The mbd will then scan the translation
unit and build all modular dependencies before returning a modified cc1 command
line such that all arguments related to modules are converted from the
implicit option to their explicit option.
This commit sets up the basic mbd infrastructure. Including the ability to
spawn a mbd and carry out a handshake between the clang invocation and mbd.
RFC: https://discourse.llvm.org/t/rfc-modules-build-daemon-build-system-agnostic-support-for-explicitly-built-modules/71524
---
.../clang/Basic/DiagnosticFrontendKinds.td | 20 ++
clang/include/clang/Basic/DiagnosticGroups.td | 1 +
clang/include/clang/Driver/Options.td | 13 +
.../include/clang/Frontend/FrontendOptions.h | 8 +
.../Tooling/ModuleBuildDaemon/Frontend.h | 36 +++
.../Tooling/ModuleBuildDaemon/SocketSupport.h | 126 +++++++++
.../clang/Tooling/ModuleBuildDaemon/Utils.h | 58 ++++
clang/lib/Driver/ToolChains/Clang.cpp | 14 +-
clang/lib/Tooling/CMakeLists.txt | 1 +
.../Tooling/ModuleBuildDaemon/CMakeLists.txt | 9 +
.../Tooling/ModuleBuildDaemon/Frontend.cpp | 166 ++++++++++++
.../ModuleBuildDaemon/SocketSupport.cpp | 66 +++++
clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp | 49 ++++
clang/test/Driver/unknown-arg.c | 2 +-
clang/test/Frontend/warning-options.cpp | 2 +-
clang/test/ModuleBuildDaemon/daemon-launch.c | 18 ++
clang/test/ModuleBuildDaemon/handshake.c | 22 ++
clang/test/lit.cfg.py | 11 +
clang/tools/driver/CMakeLists.txt | 3 +
clang/tools/driver/cc1_main.cpp | 17 +-
clang/tools/driver/cc1modbuildd_main.cpp | 255 ++++++++++++++++++
clang/tools/driver/driver.cpp | 10 +-
clang/utils/kill_process.py | 88 ++++++
llvm/include/llvm/Support/Program.h | 3 +-
llvm/include/llvm/Support/raw_socket_stream.h | 8 +-
llvm/lib/Support/Program.cpp | 9 +-
llvm/lib/Support/Unix/Program.inc | 18 +-
llvm/lib/Support/Windows/Program.inc | 7 +-
llvm/lib/Support/raw_socket_stream.cpp | 48 +++-
29 files changed, 1059 insertions(+), 29 deletions(-)
create mode 100644 clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h
create mode 100644 clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
create mode 100644 clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
create mode 100644 clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
create mode 100644 clang/test/ModuleBuildDaemon/daemon-launch.c
create mode 100644 clang/test/ModuleBuildDaemon/handshake.c
create mode 100644 clang/tools/driver/cc1modbuildd_main.cpp
create mode 100644 clang/utils/kill_process.py
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 85ecfdf9de62d4..a858bc163de0c3 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -262,6 +262,26 @@ def err_test_module_file_extension_version : Error<
"test module file extension '%0' has different version (%1.%2) than expected "
"(%3.%4)">;
+// Module Build Daemon
+def err_basepath_length :
+ Error<"BasePath '%0' is longer then the max length of %1">,
+ DefaultFatal;
+def err_mbd_handshake :
+ Error<"Attempt to handshake with module build daemon has failed: %0">,
+ DefaultFatal;
+def err_mbd_connect :
+ Error<"Attempt to connect to module build daemon has failed: %0">,
+ DefaultFatal;
+def remark_mbd_spawn :
+ Remark<"Successfully spawned module build daemon">,
+ InGroup<ModuleBuildDaemon>;
+def remark_mbd_connection :
+ Remark<"Successfully connected to module build daemon at %0">,
+ InGroup<ModuleBuildDaemon>;
+def remark_mbd_handshake :
+ Remark<"Successfully completed handshake with module build daemon">,
+ InGroup<ModuleBuildDaemon>;
+
def warn_eagerly_load_for_standard_cplusplus_modules : Warning<
"the form '-fmodule-file=<BMI-path>' is deprecated for standard C++ named modules;"
"consider to use '-fmodule-file=<module-name>=<BMI-path>' instead">,
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 6765721ae7002c..59fdae48f8f6e9 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -517,6 +517,7 @@ def MismatchedTags : DiagGroup<"mismatched-tags">;
def MissingFieldInitializers : DiagGroup<"missing-field-initializers">;
def ModuleLock : DiagGroup<"module-lock">;
def ModuleBuild : DiagGroup<"module-build">;
+def ModuleBuildDaemon : DiagGroup<"module-build-daemon">;
def ModuleImport : DiagGroup<"module-import">;
def ModuleConflict : DiagGroup<"module-conflict">;
def ModuleFileExtension : DiagGroup<"module-file-extension">;
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 4b232b8aab722a..376b7a313bebee 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2960,6 +2960,19 @@ defm declspec : BoolOption<"f", "declspec",
NegFlag<SetFalse, [], [ClangOption], "Disallow">,
BothFlags<[], [ClangOption, CC1Option],
" __declspec as a keyword">>, Group<f_clang_Group>;
+
+def fmodule_build_daemon : Flag<["-"], "fmodule-build-daemon">, Group<f_Group>,
+ Flags<[NoXarchOption]>,
+ Visibility<[ClangOption, CC1Option]>,
+ HelpText<"Enables module build daemon functionality">,
+ MarshallingInfoFlag<FrontendOpts<"ModuleBuildDaemon">>;
+def fmodule_build_daemon_EQ : Joined<["-"], "fmodule-build-daemon=">, Group<f_Group>,
+ Flags<[NoXarchOption]>,
+ Visibility<[ClangOption, CC1Option]>,
+ HelpText<"Enables module build daemon functionality and defines the path to "
+ "where the module build daemon will write log and socket files">,
+ MarshallingInfoString<FrontendOpts<"ModuleBuildDaemonPath">>;
+
def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">, Group<i_Group>,
Flags<[]>, Visibility<[ClangOption, CC1Option]>,
MetaVarName<"<directory>">,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 53a8681cfdbba0..607f7a21cb9702 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -383,6 +383,10 @@ class FrontendOptions {
LLVM_PREFERRED_TYPE(bool)
unsigned ModulesShareFileManager : 1;
+ /// Connect to module build daemon.
+ LLVM_PREFERRED_TYPE(bool)
+ unsigned ModuleBuildDaemon : 1;
+
CodeCompleteOptions CodeCompleteOpts;
/// Specifies the output format of the AST.
@@ -468,6 +472,10 @@ class FrontendOptions {
/// The output file, if any.
std::string OutputFile;
+ /// If given, the path to the module build daemon's output files and socket
+ /// address
+ std::string ModuleBuildDaemonPath;
+
/// If given, the new suffix for fix-it rewritten files.
std::string FixItSuffix;
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h b/clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h
new file mode 100644
index 00000000000000..727738ef77b1bd
--- /dev/null
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h
@@ -0,0 +1,36 @@
+//===----------------------------- Frontend.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_FRONTEND_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_FRONTEND_H
+
+#include "clang/Frontend/CompilerInvocation.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+namespace clang::tooling::cc1modbuildd {
+
+llvm::Error attemptHandshake(llvm::raw_socket_stream &Client,
+ clang::DiagnosticsEngine &Diag);
+
+llvm::Error spawnModuleBuildDaemon(const clang::CompilerInvocation &Clang,
+ const char *Argv0,
+ clang::DiagnosticsEngine &Diag,
+ std::string BasePath);
+
+llvm::Expected<std::unique_ptr<llvm::raw_socket_stream>>
+getModuleBuildDaemon(const clang::CompilerInvocation &Clang, const char *Argv0,
+ clang::DiagnosticsEngine &Diag, llvm::StringRef BasePath);
+
+void spawnModuleBuildDaemonAndHandshake(const clang::CompilerInvocation &Clang,
+ const char *Argv0,
+ clang::DiagnosticsEngine &Diag);
+
+} // namespace clang::tooling::cc1modbuildd
+
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_CLIENT_H
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h b/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
new file mode 100644
index 00000000000000..94a10359c61471
--- /dev/null
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h
@@ -0,0 +1,126 @@
+//===------------------------- SocketMsgSupport.h -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+namespace clang::tooling::cc1modbuildd {
+
+enum class ActionType { HANDSHAKE };
+enum class StatusType { REQUEST, SUCCESS, FAILURE };
+
+struct BaseMsg {
+ ActionType MsgAction;
+ StatusType MsgStatus;
+
+ BaseMsg() = default;
+ BaseMsg(ActionType Action, StatusType Status)
+ : MsgAction(Action), MsgStatus(Status) {}
+};
+
+struct HandshakeMsg : public BaseMsg {
+ HandshakeMsg() = default;
+ HandshakeMsg(ActionType Action, StatusType Status)
+ : BaseMsg(Action, Status) {}
+};
+
+llvm::Expected<std::unique_ptr<llvm::raw_socket_stream>>
+connectToSocket(llvm::StringRef SocketPath);
+llvm::Expected<std::string>
+readBufferFromSocket(llvm::raw_socket_stream &Socket);
+llvm::Error writeBufferToSocket(llvm::raw_socket_stream &Socket,
+ llvm::StringRef Buffer);
+
+template <typename T> std::string convertMsgStructToBuffer(T MsgStruct) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ std::string Buffer;
+ llvm::raw_string_ostream OS(Buffer);
+ llvm::yaml::Output YamlOut(OS);
+
+ // TODO confirm yaml::Output does not have any error messages
+ YamlOut << MsgStruct;
+
+ return Buffer;
+}
+
+template <typename T>
+llvm::Expected<T> convertBufferToMsgStruct(llvm::StringRef Buffer) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ T MsgStruct;
+ llvm::yaml::Input YamlIn(Buffer);
+ YamlIn >> MsgStruct;
+
+ // YamlIn.error() dumps an error message if there is one
+ if (YamlIn.error())
+ return llvm::make_error<llvm::StringError>(
+ "Syntax or semantic error during YAML parsing",
+ llvm::inconvertibleErrorCode());
+
+ return MsgStruct;
+}
+
+template <typename T>
+llvm::Expected<T> readMsgStructFromSocket(llvm::raw_socket_stream &Socket) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ llvm::Expected<std::string> MaybeBuffer = readBufferFromSocket(Socket);
+ if (!MaybeBuffer)
+ return std::move(MaybeBuffer.takeError());
+ std::string Buffer = std::move(*MaybeBuffer);
+
+ llvm::Expected<T> MaybeMsgStruct = convertBufferToMsgStruct<T>(Buffer);
+ if (!MaybeMsgStruct)
+ return std::move(MaybeMsgStruct.takeError());
+ return std::move(*MaybeMsgStruct);
+}
+
+template <typename T>
+llvm::Error writeMsgStructToSocket(llvm::raw_socket_stream &Socket,
+ T MsgStruct) {
+ static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+ std::string Buffer = convertMsgStructToBuffer(MsgStruct);
+ if (llvm::Error Err = writeBufferToSocket(Socket, Buffer))
+ return std::move(Err);
+ return llvm::Error::success();
+}
+
+} // namespace clang::tooling::cc1modbuildd
+
+namespace cc1mod = clang::tooling::cc1modbuildd;
+
+template <> struct llvm::yaml::ScalarEnumerationTraits<cc1mod::StatusType> {
+ static void enumeration(IO &Io, cc1mod::StatusType &Value) {
+ Io.enumCase(Value, "REQUEST", cc1mod::StatusType::REQUEST);
+ Io.enumCase(Value, "SUCCESS", cc1mod::StatusType::SUCCESS);
+ Io.enumCase(Value, "FAILURE", cc1mod::StatusType::FAILURE);
+ }
+};
+
+template <> struct llvm::yaml::ScalarEnumerationTraits<cc1mod::ActionType> {
+ static void enumeration(IO &Io, cc1mod::ActionType &Value) {
+ Io.enumCase(Value, "HANDSHAKE", cc1mod::ActionType::HANDSHAKE);
+ }
+};
+
+template <> struct llvm::yaml::MappingTraits<cc1mod::HandshakeMsg> {
+ static void mapping(IO &Io, cc1mod::HandshakeMsg &Info) {
+ Io.mapRequired("Action", Info.MsgAction);
+ Io.mapRequired("Status", Info.MsgStatus);
+ }
+};
+
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
diff --git a/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
new file mode 100644
index 00000000000000..77fa4aee1e2ca9
--- /dev/null
+++ b/clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h
@@ -0,0 +1,58 @@
+//===------------------------------ Utils.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Functions required by both the frontend and the module build daemon
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_UTILS_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_UTILS_H
+
+#include "llvm/Support/Error.h"
+
+#include <string>
+
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+// winsock2.h must be included before afunix.h
+// clang-format off
+#include <winsock2.h>
+#include <afunix.h>
+// clang-format on
+#else
+#include <sys/un.h>
+#endif
+
+namespace clang::tooling::cc1modbuildd {
+
+constexpr std::string_view SOCKET_FILE_NAME = "mbd.sock";
+constexpr std::string_view STDOUT_FILE_NAME = "mbd.out";
+constexpr std::string_view STDERR_FILE_NAME = "mbd.err";
+constexpr std::string_view MODULE_BUILD_DAEMON_FLAG = "-cc1modbuildd";
+
+// A llvm::raw_socket_stream uses sockaddr_un
+constexpr size_t SOCKET_ADDR_MAX_LENGTH = sizeof(sockaddr_un::sun_path);
+
+constexpr size_t BASEPATH_MAX_LENGTH =
+ SOCKET_ADDR_MAX_LENGTH - SOCKET_FILE_NAME.length();
+
+// How long should the module build daemon sit ideal before exiting
+constexpr int TimeoutSec = 15;
+
+// Get a temprary location where the daemon can store log files and a socket
+// address. Of the format /tmp/clang-<BLAKE3HashOfClangFullVersion>/
+std::string getBasePath();
+
+// Check if the user provided BasePath is short enough
+bool validBasePathLength(llvm::StringRef);
+
+} // namespace clang::tooling::cc1modbuildd
+
+#endif
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 13bf2421154372..762f760af84228 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-
#include "Clang.h"
#include "AMDGPU.h"
#include "Arch/AArch64.h"
@@ -3764,6 +3763,19 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
bool HaveStdCXXModules = IsCXX && HaveStd20;
bool HaveModules = HaveStdCXXModules;
+ // -fmodule-build-daemon enables module build daemon functionality
+ if (Args.hasArg(options::OPT_fmodule_build_daemon))
+ Args.AddLastArg(CmdArgs, options::OPT_fmodule_build_daemon);
+
+ // by default module build daemon socket address and output files are saved
+ // under /tmp/ but that can be overridden by providing the
+ // -fmodule-build-daemon=<path> flag
+ if (Arg *A = Args.getLastArg(options::OPT_fmodule_build_daemon_EQ)) {
+ CmdArgs.push_back(
+ Args.MakeArgString(Twine("-fmodule-build-daemon=") + A->getValue()));
+ CmdArgs.push_back("-fmodule-build-daemon");
+ }
+
// -fmodules enables the use of precompiled modules (off by default).
// Users can pass -fno-cxx-modules to turn off modules support for
// C++/Objective-C++ programs.
diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt
index aff39e4de13c0b..85752e57733265 100644
--- a/clang/lib/Tooling/CMakeLists.txt
+++ b/clang/lib/Tooling/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(DumpTool)
add_subdirectory(Syntax)
add_subdirectory(DependencyScanning)
add_subdirectory(Transformer)
+add_subdirectory(ModuleBuildDaemon)
# Replace the last lib component of the current binary directory with include
string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE)
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt b/clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
new file mode 100644
index 00000000000000..c043557206ef9e
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt
@@ -0,0 +1,9 @@
+set(LLVM_LINK_COMPONENTS
+ Support
+ )
+
+add_clang_library(clangModuleBuildDaemon
+ Frontend.cpp
+ SocketSupport.cpp
+ Utils.cpp
+ )
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
new file mode 100644
index 00000000000000..cf73e27f246bba
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Frontend.cpp
@@ -0,0 +1,166 @@
+//===---------------------------- Frontend.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/Frontend.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h"
+#include "clang/Tooling/ModuleBuildDaemon/Utils.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Program.h"
+
+#include <cerrno>
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <optional>
+#include <string>
+#include <thread>
+
+using namespace llvm;
+
+namespace clang::tooling::cc1modbuildd {
+
+llvm::Error attemptHandshake(raw_socket_stream &Client,
+ DiagnosticsEngine &Diag) {
+
+ // Send HandshakeMsg to module build daemon
+ HandshakeMsg Request{ActionType::HANDSHAKE, StatusType::REQUEST};
+ if (llvm::Error Err = writeMsgStructToSocket(Client, Request))
+ return std::move(Err);
+
+ // Read response from module build daemon
+ Expected<HandshakeMsg> MaybeResponse =
+ readMsgStructFromSocket<HandshakeMsg>(Client);
+ if (!MaybeResponse) {
+ return std::move(MaybeResponse.takeError());
+ }
+ HandshakeMsg Response = std::move(*MaybeResponse);
+
+ assert(Response.MsgAction == ActionType::HANDSHAKE &&
+ "The response ActionType should only ever be HANDSHAKE");
+
+ if (Response.MsgStatus == StatusType::SUCCESS) {
+ return llvm::Error::success();
+ }
+
+ return llvm::make_error<StringError>(
+ "Received handshake response 'FAILURE' from module build daemon",
+ std::make_error_code(std::errc::operation_not_permitted));
+}
+
+llvm::Error spawnModuleBuildDaemon(const CompilerInvocation &Clang,
+ const char *Argv0, DiagnosticsEngine &Diag,
+ std::string BasePath) {
+
+ std::vector<StringRef> Args = {Argv0, MODULE_BUILD_DAEMON_FLAG};
+ if (!Clang.getFrontendOpts().ModuleBuildDaemonPath.empty())
+ Args.push_back(BasePath.c_str());
+
+ std::string ErrorBuffer;
+ llvm::sys::ExecuteNoWait(Argv0, Args, std::nullopt, {}, 0, &ErrorBuffer,
+ nullptr, nullptr, /*DetachProcess*/ true);
+
+ // llvm::sys::ExecuteNoWait can fail for a variety of reasons which can't be
+ // generalized to one error code
+ if (!ErrorBuffer.empty())
+ return llvm::make_error<StringError>(ErrorBuffer, inconvertibleErrorCode());
+
+ Diag.Report(diag::remark_mbd_spawn);
+ return llvm::Error::success();
+}
+
+Expected<std::unique_ptr<raw_socket_stream>>
+getModuleBuildDaemon(const CompilerInvocation &Clang, const char *Argv0,
+ DiagnosticsEngine &Diag, StringRef BasePath) {
+
+ SmallString<128> SocketPath = BasePath;
+ llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+
+ if (llvm::sys::fs::exists(SocketPath)) {
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ connectToSocket(SocketPath);
+ if (MaybeClient) {
+ return std::move(*MaybeClient);
+ }
+ consumeError(MaybeClient.takeError());
+ }
+
+ if (llvm::Error Err =
+ spawnModuleBuildDaemon(Clang, Argv0, Diag, BasePath.str()))
+ return std::move(Err);
+
+ constexpr unsigned int MICROSEC_IN_SEC = 1000000;
+ constexpr unsigned int MAX_WAIT_TIME = 30 * MICROSEC_IN_SEC;
+ unsigned int CumulativeTime = 0;
+ unsigned int WaitTime = 10;
+
+ while (CumulativeTime <= MAX_WAIT_TIME) {
+ // Wait a bit then check to see if the module build daemon has initialized
+ std::this_thread::sleep_for(std::chrono::microseconds(WaitTime));
+
+ if (llvm::sys::fs::exists(SocketPath)) {
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ connectToSocket(SocketPath);
+ if (MaybeClient) {
+ Diag.Report(diag::remark_mbd_connection) << SocketPath;
+ return std::move(*MaybeClient);
+ }
+ consumeError(MaybeClient.takeError());
+ }
+
+ CumulativeTime += WaitTime;
+ WaitTime = WaitTime * 2;
+ }
+
+ // After waiting around 30 seconds give up and return an error
+ return llvm::make_error<StringError>(
+ "Max wait time exceeded: ",
+ std::make_error_code(std::errc::no_such_process));
+}
+
+void spawnModuleBuildDaemonAndHandshake(const CompilerInvocation &Clang,
+ const char *Argv0,
+ DiagnosticsEngine &Diag) {
+
+ // The module build daemon stores all output files and its socket address
+ // under BasePath. Either set BasePath to a user provided option or create an
+ // appropriate BasePath based on the BLAKE3 hash of the full clang version
+ std::string BasePath;
+ if (Clang.getFrontendOpts().ModuleBuildDaemonPath.empty())
+ BasePath = getBasePath();
+ else {
+ // Get user provided BasePath and confirm it is short enough
+ BasePath = Clang.getFrontendOpts().ModuleBuildDaemonPath;
+ if (!validBasePathLength(BasePath)) {
+ Diag.Report(diag::err_basepath_length) << BasePath << BASEPATH_MAX_LENGTH;
+ return;
+ }
+ }
+
+ // If module build daemon does not exist spawn module build daemon
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ getModuleBuildDaemon(Clang, Argv0, Diag, BasePath);
+ if (!MaybeClient) {
+ Diag.Report(diag::err_mbd_connect) << MaybeClient.takeError();
+ return;
+ }
+ raw_socket_stream &Client = **MaybeClient;
+ if (llvm::Error HandshakeErr = attemptHandshake(Client, Diag)) {
+ Diag.Report(diag::err_mbd_handshake) << std::move(HandshakeErr);
+ return;
+ }
+
+ Diag.Report(diag::remark_mbd_handshake);
+ return;
+}
+
+} // namespace clang::tooling::cc1modbuildd
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp b/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
new file mode 100644
index 00000000000000..08ddfff8c7f387
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp
@@ -0,0 +1,66 @@
+//===------------------------ SocketMsgSupport.cpp ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+#include <memory>
+#include <string>
+
+using namespace llvm;
+
+namespace clang::tooling::cc1modbuildd {
+
+Expected<std::unique_ptr<raw_socket_stream>>
+connectToSocket(StringRef SocketPath) {
+
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+ raw_socket_stream::createConnectedUnix(SocketPath);
+ if (!MaybeClient)
+ return std::move(MaybeClient.takeError());
+
+ return std::move(*MaybeClient);
+}
+
+Expected<std::string> readBufferFromSocket(raw_socket_stream &Socket) {
+
+ constexpr unsigned short MAX_BUFFER = 4096;
+ char Buffer[MAX_BUFFER];
+ std::string ReturnBuffer;
+
+ ssize_t n = 0;
+ while ((n = Socket.read(Buffer, MAX_BUFFER)) > 0) {
+ ReturnBuffer.append(Buffer, n);
+ // Read until \n... encountered which is the last line of a YAML document
+ if (ReturnBuffer.find("\n...") != std::string::npos)
+ break;
+ }
+
+ if (Socket.has_error()) {
+ std::error_code EC = Socket.error();
+ Socket.clear_error();
+ return make_error<StringError>("Failed socket read: ", EC);
+ }
+ return ReturnBuffer;
+}
+
+Error writeBufferToSocket(raw_socket_stream &Socket, StringRef Buffer) {
+ Socket << Buffer;
+
+ if (Socket.has_error()) {
+ std::error_code EC = Socket.error();
+ Socket.clear_error();
+ return make_error<StringError>("Failed socket write: ", EC);
+ }
+
+ Socket.flush();
+ return Error::success();
+}
+
+} // namespace clang::tooling::cc1modbuildd
diff --git a/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp b/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
new file mode 100644
index 00000000000000..93fc14c4ba23dc
--- /dev/null
+++ b/clang/lib/Tooling/ModuleBuildDaemon/Utils.cpp
@@ -0,0 +1,49 @@
+//===------------------------------ Utils.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/Utils.h"
+#include "clang/Basic/Version.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/BLAKE3.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Path.h"
+
+#include <string>
+
+using namespace llvm;
+
+namespace clang::tooling::cc1modbuildd {
+
+std::string getBasePath() {
+ llvm::BLAKE3 Hash;
+ Hash.update(clang::getClangFullVersion());
+ auto HashResult = Hash.final<sizeof(uint64_t)>();
+ uint64_t HashValue =
+ llvm::support::endian::read<uint64_t, llvm::endianness::native>(
+ HashResult.data());
+ std::string Key = toString(llvm::APInt(64, HashValue), 36, /*Signed*/ false);
+
+ // Set paths
+ SmallString<128> BasePath;
+ llvm::sys::path::system_temp_directory(/*erasedOnReboot*/ true, BasePath);
+ llvm::sys::path::append(BasePath, "clang-" + Key);
+ return BasePath.c_str();
+}
+
+bool validBasePathLength(llvm::StringRef Address) {
+ // Confirm that the user provided BasePath is short enough to allow the socket
+ // address to fit within the alloted space
+ if (Address.str().length() > BASEPATH_MAX_LENGTH) {
+ return false;
+ }
+ return true;
+}
+
+} // namespace clang::tooling::cc1modbuildd
\ No newline at end of file
diff --git a/clang/test/Driver/unknown-arg.c b/clang/test/Driver/unknown-arg.c
index 52ea0f5ff3220f..3a22b824adc5c7 100644
--- a/clang/test/Driver/unknown-arg.c
+++ b/clang/test/Driver/unknown-arg.c
@@ -59,7 +59,7 @@
// SILENT-NOT: warning:
// CC1AS-DID-YOU-MEAN: error: unknown argument '-hell'; did you mean '-help'?
// CC1AS-DID-YOU-MEAN: error: unknown argument '--version'; did you mean '-version'?
-// UNKNOWN-INTEGRATED: error: unknown integrated tool '-cc1asphalt'. Valid tools include '-cc1' and '-cc1as'.
+// UNKNOWN-INTEGRATED: error: unknown integrated tool '-cc1asphalt'. Valid tools include '-cc1', '-cc1as', '-cc1gen-reproducer', and '-cc1modbuildd'.
// RUN: %clang -S %s -o %t.s -Wunknown-to-clang-option 2>&1 | FileCheck --check-prefix=IGNORED %s
diff --git a/clang/test/Frontend/warning-options.cpp b/clang/test/Frontend/warning-options.cpp
index 444733c8b7f365..42a7109b8d48c5 100644
--- a/clang/test/Frontend/warning-options.cpp
+++ b/clang/test/Frontend/warning-options.cpp
@@ -3,6 +3,6 @@
// CHECK: unknown warning option '-Wmonkey'
// CHECK: unknown warning option '-Wno-monkey'
// CHECK: unknown warning option '-Wno-unused-command-line-arguments'; did you mean '-Wno-unused-command-line-argument'?
-// CHECK: unknown warning option '-Wmodule-build'; did you mean '-Wmodule-conflict'?
+// CHECK: unknown warning option '-Wmodule-build'
// CHECK-NEXT: unknown -Werror warning specifier: '-Werror-vla'
// CHECK: unknown remark option '-Rmodule-built'; did you mean '-Rmodule-build'?
diff --git a/clang/test/ModuleBuildDaemon/daemon-launch.c b/clang/test/ModuleBuildDaemon/daemon-launch.c
new file mode 100644
index 00000000000000..2bfea6aec9dccd
--- /dev/null
+++ b/clang/test/ModuleBuildDaemon/daemon-launch.c
@@ -0,0 +1,18 @@
+// Check that the module build daemon can create a unix socket
+
+// RUN: rm -rf mbd-launch %t
+
+// timeout should exit with status 124 which is treated as a failure by lit on
+// windows. Ideally we would be like to check the exit code and only return true
+// if it equals 124 but lit does not support global bash sysmbols like $?
+
+// RUN: timeout --signal=SIGTERM 2 %clang -cc1modbuildd mbd-launch -v || true
+// RUN: cat mbd-launch/mbd.out | sed 's:\\\\\?:/:g' | FileCheck %s
+
+// CHECK: mbd created and binded to socket at: mbd-launch/mbd.sock
+
+// Make sure mbd.sock does not exist
+// RUN: [ ! -f "mbd-launch/mbd.sock" ] && true || false
+
+// Make sure mbd.err is empty
+// RUN: [ ! -s "mbd-launch/mbd.err" ] && true || false
diff --git a/clang/test/ModuleBuildDaemon/handshake.c b/clang/test/ModuleBuildDaemon/handshake.c
new file mode 100644
index 00000000000000..059ecf6fae188c
--- /dev/null
+++ b/clang/test/ModuleBuildDaemon/handshake.c
@@ -0,0 +1,22 @@
+// Check that a clang invocation can spawn and handshake with a module build daemon
+
+// RUN: %kill-process "-cc1modbuildd mbd-handshake"
+// RUN: rm -rf mbd-handshake %t
+// RUN: split-file %s %t
+
+//--- main.c
+int main() {return 0;}
+
+// RUN: %clang -fmodule-build-daemon=mbd-handshake -Rmodule-build-daemon %t/main.c &> %t/output-new || true
+// RUN: %clang -fmodule-build-daemon=mbd-handshake -Rmodule-build-daemon %t/main.c &> %t/output-existing || true
+// RUN: %kill-process "-cc1modbuildd mbd-handshake"
+
+// RUN: cat %t/output-new | sed 's:\\\\\?:/:g' | FileCheck %s
+// RUN: cat %t/output-existing | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-EXIST
+
+// CHECK: remark: Successfully spawned module build daemon [-Rmodule-build-daemon]
+// CHECK: remark: Successfully connected to module build daemon at mbd-handshake/mbd.sock [-Rmodule-build-daemon]
+// CHECK: remark: Successfully completed handshake with module build daemon [-Rmodule-build-daemon]
+
+// Check that a clang invocation can handshake with an existing module build daemon
+// CHECK-EXIST: remark: Successfully completed handshake with module build daemon [-Rmodule-build-daemon]
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 271372b928ac55..7a51d278da6577 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -203,6 +203,17 @@ def have_host_clang_repl_cuda():
)
)
+config.substitutions.append(
+ (
+ "%kill-process",
+ '"%s" %s'
+ % (
+ config.python_executable,
+ os.path.join(config.clang_src_dir, "utils", "kill_process.py"),
+ ),
+ )
+)
+
config.substitutions.append(("%host_cc", config.host_cc))
config.substitutions.append(("%host_cxx", config.host_cxx))
diff --git a/clang/tools/driver/CMakeLists.txt b/clang/tools/driver/CMakeLists.txt
index d70b92b0984e52..e44475b58b8ac4 100644
--- a/clang/tools/driver/CMakeLists.txt
+++ b/clang/tools/driver/CMakeLists.txt
@@ -28,6 +28,7 @@ add_clang_tool(clang
cc1_main.cpp
cc1as_main.cpp
cc1gen_reproducer_main.cpp
+ cc1modbuildd_main.cpp
DEPENDS
intrinsics_gen
@@ -39,9 +40,11 @@ clang_target_link_libraries(clang
PRIVATE
clangBasic
clangCodeGen
+ clangDependencyScanning
clangDriver
clangFrontend
clangFrontendTool
+ clangModuleBuildDaemon
clangSerialization
)
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index e9d2c6aad371db..b07801a2dbc21a 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -25,6 +25,7 @@
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/FrontendTool/Utils.h"
+#include "clang/Tooling/ModuleBuildDaemon/Frontend.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LinkAllPasses.h"
@@ -63,7 +64,7 @@ using namespace llvm::opt;
static void LLVMErrorHandler(void *UserData, const char *Message,
bool GenCrashDiag) {
- DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
+ DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine *>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
@@ -201,7 +202,7 @@ static int PrintSupportedExtensions(std::string TargetStr) {
const llvm::Triple &MachineTriple = TheTargetMachine->getTargetTriple();
const llvm::MCSubtargetInfo *MCInfo = TheTargetMachine->getMCSubtargetInfo();
const llvm::ArrayRef<llvm::SubtargetFeatureKV> Features =
- MCInfo->getAllProcessorFeatures();
+ MCInfo->getAllProcessorFeatures();
llvm::StringMap<llvm::StringRef> DescMap;
for (const llvm::SubtargetFeatureKV &feature : Features)
@@ -270,7 +271,7 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
Clang->getHeaderSearchOpts().ResourceDir.empty())
Clang->getHeaderSearchOpts().ResourceDir =
- CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
+ CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
// Create the actual diagnostics engine.
Clang->createDiagnostics();
@@ -279,8 +280,8 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
- llvm::install_fatal_error_handler(LLVMErrorHandler,
- static_cast<void*>(&Clang->getDiagnostics()));
+ llvm::install_fatal_error_handler(
+ LLVMErrorHandler, static_cast<void *>(&Clang->getDiagnostics()));
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
if (!Success) {
@@ -288,6 +289,12 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
return 1;
}
+ // Handle module build daemon functionality if enabled
+ if (Clang->getFrontendOpts().ModuleBuildDaemon) {
+ clang::tooling::cc1modbuildd::spawnModuleBuildDaemonAndHandshake(
+ Clang->getInvocation(), Argv0, Clang->getDiagnostics());
+ }
+
// Execute the frontend actions.
{
llvm::TimeTraceScope TimeScope("ExecuteCompiler");
diff --git a/clang/tools/driver/cc1modbuildd_main.cpp b/clang/tools/driver/cc1modbuildd_main.cpp
new file mode 100644
index 00000000000000..61b9383bf1547d
--- /dev/null
+++ b/clang/tools/driver/cc1modbuildd_main.cpp
@@ -0,0 +1,255 @@
+//===------- cc1modbuildd_main.cpp - Clang CC1 Module Build Daemon --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h"
+#include "clang/Tooling/ModuleBuildDaemon/Utils.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
+
+#include <csignal>
+#include <cstdbool>
+#include <fstream>
+#include <optional>
+#include <string>
+#include <system_error>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+using namespace llvm;
+using namespace clang::tooling::cc1modbuildd;
+
+// Create unbuffered STDOUT stream so that any logging done by the module build
+// daemon can be viewed without having to terminate the process
+static raw_fd_ostream &unbuff_outs() {
+ static raw_fd_ostream S(fileno(stdout), false, true);
+ return S;
+}
+
+static bool VerboseLog = false;
+static void verboseLog(const llvm::Twine &message) {
+ if (VerboseLog) {
+ unbuff_outs() << message << '\n';
+ }
+}
+
+namespace {
+
+class ModuleBuildDaemonServer {
+public:
+ SmallString<256> SocketPath;
+ SmallString<256> STDERR;
+ SmallString<256> STDOUT;
+
+ ModuleBuildDaemonServer(StringRef Path)
+ : SocketPath(Path), STDERR(Path), STDOUT(Path) {
+ llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME);
+ llvm::sys::path::append(STDOUT, STDOUT_FILE_NAME);
+ llvm::sys::path::append(STDERR, STDERR_FILE_NAME);
+ }
+
+ ~ModuleBuildDaemonServer() { shutdownDaemon(); }
+
+ int setupDaemonEnv();
+ int createDaemonSocket();
+ int listenForClients();
+
+ static void handleConnection(std::shared_ptr<raw_socket_stream> Connection);
+
+ // TODO: modify so when shutdownDaemon is called the daemon stops accepting
+ // new client connections and waits for all existing client connections to
+ // terminate before closing the file descriptor and exiting
+ void shutdownDaemon() {
+ RunServiceLoop = false;
+ if(ServerListener.has_value())
+ ServerListener.value().shutdown();
+ }
+
+private:
+ std::atomic<bool> RunServiceLoop = true;
+ std::optional<llvm::ListeningSocket> ServerListener;
+};
+
+// Used to handle signals
+ModuleBuildDaemonServer *DaemonPtr = nullptr;
+void handleSignal(int) {
+ DaemonPtr->shutdownDaemon();
+ exit(EXIT_SUCCESS);
+}
+} // namespace
+
+// Sets up file descriptors and signals for module build daemon
+int ModuleBuildDaemonServer::setupDaemonEnv() {
+
+#ifdef _WIN32
+ freopen("NUL", "r", stdin);
+#else
+ close(STDIN_FILENO);
+#endif
+
+ freopen(STDOUT.c_str(), "a", stdout);
+ freopen(STDERR.c_str(), "a", stderr);
+
+ if (std::signal(SIGTERM, handleSignal) == SIG_ERR) {
+ errs() << "failed to handle SIGTERM" << '\n';
+ exit(EXIT_FAILURE);
+ }
+
+ if (std::signal(SIGINT, handleSignal) == SIG_ERR) {
+ errs() << "failed to handle SIGINT" << '\n';
+ exit(EXIT_FAILURE);
+ }
+
+// TODO: Figure out how to do this on windows
+#ifdef SIGHUP
+ if (::signal(SIGHUP, SIG_IGN) == SIG_ERR) {
+ errs() << "failed to handle SIGHUP" << '\n';
+ exit(EXIT_FAILURE);
+ }
+#endif
+
+ return EXIT_SUCCESS;
+}
+
+// Creates unix socket for IPC with frontends
+int ModuleBuildDaemonServer::createDaemonSocket() {
+
+ Expected<ListeningSocket> MaybeServerListener =
+ llvm::ListeningSocket::createUnix(SocketPath);
+
+ if (llvm::Error Err = MaybeServerListener.takeError()) {
+ llvm::handleAllErrors(std::move(Err), [&](const llvm::StringError &SE) {
+ std::error_code EC = SE.convertToErrorCode();
+ // Exit successfully if the socket address is already in use. When
+ // translation units are compiled in parallel, until the socket file is
+ // created, all clang invocations will try to spawn a module build daemon.
+#ifdef _WIN32
+ if (EC.value() == WSAEADDRINUSE) {
+#else
+ if (EC == std::errc::address_in_use) {
+#endif
+ exit(EXIT_SUCCESS);
+ } else {
+ llvm::errs() << "MBD failed to create unix socket: " << SE.message()
+ << EC.message() << '\n';
+ exit(EXIT_FAILURE);
+ }
+ });
+ }
+
+ verboseLog("mbd created and binded to socket at: " + SocketPath);
+ ServerListener.emplace(std::move(*MaybeServerListener));
+ return 0;
+}
+
+// Function submitted to thread pool with each frontend connection. Not
+// responsible for closing frontend socket connections
+void ModuleBuildDaemonServer::handleConnection(
+ std::shared_ptr<llvm::raw_socket_stream> MovableConnection) {
+
+ llvm::raw_socket_stream &Connection = *MovableConnection;
+
+ // Read request from frontend
+ Expected<HandshakeMsg> MaybeHandshakeMsg =
+ readMsgStructFromSocket<HandshakeMsg>(Connection);
+ if (!MaybeHandshakeMsg) {
+ errs() << "MBD failed to read frontend request: "
+ << llvm::toString(MaybeHandshakeMsg.takeError()) << '\n';
+ return;
+ }
+
+ // Send response to frontend
+ HandshakeMsg Msg(ActionType::HANDSHAKE, StatusType::SUCCESS);
+ if (llvm::Error WriteErr = writeMsgStructToSocket(Connection, Msg)) {
+ errs() << "MBD failed to respond to frontend request: "
+ << llvm::toString(std::move(WriteErr)) << '\n';
+ return;
+ }
+ return;
+}
+
+int ModuleBuildDaemonServer::listenForClients() {
+
+ llvm::ThreadPool Pool;
+
+ while (RunServiceLoop) {
+ Expected<std::unique_ptr<raw_socket_stream>> MaybeConnection =
+ ServerListener.value().accept({TimeoutSec, 0});
+
+ if (llvm::Error Err = MaybeConnection.takeError()) {
+
+ llvm::handleAllErrors(std::move(Err), [&](const llvm::StringError &SE) {
+ std::error_code EC = SE.convertToErrorCode();
+ if (EC == std::errc::timed_out)
+ RunServiceLoop = false;
+ else
+ errs() << "MBD failed to accept incoming connection: "
+ << SE.getMessage() << ": " << EC.message() << '\n';
+ });
+
+ continue;
+ }
+
+ std::shared_ptr<raw_socket_stream> Connection(std::move(*MaybeConnection));
+ Pool.async(handleConnection, Connection);
+ }
+ return 0;
+}
+
+// Module build daemon is spawned with the following command line:
+//
+// clang -cc1modbuildd [<path>] [-v]
+//
+// OPTIONS
+// <path>
+// Specifies the path to all the files created by the module build daemon.
+// If provided, <path> should immediately follow -cc1modbuildd.
+//
+// -v
+// Provides verbose debug information.
+//
+// NOTES
+// The arguments <path> and -v are optional. If <path> is not provided then
+// BasePath will be /tmp/clang-<BLAKE3HashOfClangFullVersion>
+//
+int cc1modbuildd_main(ArrayRef<const char *> Argv) {
+
+ // -cc1modbuildd is sliced away when Argv is pased to cc1modbuildd_main
+ if (find(Argv, StringRef("-v")) != Argv.end())
+ VerboseLog = true;
+
+ std::string BasePath;
+ // If an argument exists and it is not -v then it must be a BasePath
+ if (!Argv.empty() && strcmp(Argv[0], "-v") != 0)
+ BasePath = Argv[0];
+ else
+ BasePath = getBasePath();
+
+ if (!validBasePathLength(BasePath)) {
+ errs() << "BasePath '" << BasePath << "' is longer then the max length of "
+ << std::to_string(BASEPATH_MAX_LENGTH) << '\n';
+ return 1;
+ }
+
+ llvm::sys::fs::create_directories(BasePath);
+ ModuleBuildDaemonServer Daemon(BasePath);
+
+ // Used to handle signals
+ DaemonPtr = &Daemon;
+
+ Daemon.setupDaemonEnv();
+ Daemon.createDaemonSocket();
+ Daemon.listenForClients();
+
+ return EXIT_SUCCESS;
+}
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index 1407c7fcdab769..48c480ca7f9b60 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -212,6 +212,7 @@ extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
const char *Argv0, void *MainAddr,
const llvm::ToolContext &);
+extern int cc1modbuildd_main(ArrayRef<const char *> Argv);
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
SmallVectorImpl<const char *> &ArgVector,
@@ -368,9 +369,14 @@ static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV,
if (Tool == "-cc1gen-reproducer")
return cc1gen_reproducer_main(ArrayRef(ArgV).slice(2), ArgV[0],
GetExecutablePathVP, ToolContext);
- // Reject unknown tools.
+ if (Tool == "-cc1modbuildd") {
+ return cc1modbuildd_main(ArrayRef(ArgV).slice(2));
+ }
+
+ // Reject unknown tools
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
- << "Valid tools include '-cc1' and '-cc1as'.\n";
+ << "Valid tools include '-cc1', '-cc1as', '-cc1gen-reproducer', "
+ "and '-cc1modbuildd'.\n";
return 1;
}
diff --git a/clang/utils/kill_process.py b/clang/utils/kill_process.py
new file mode 100644
index 00000000000000..ec4a343af8256b
--- /dev/null
+++ b/clang/utils/kill_process.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+# Kills all processes whos command line match provided pattern
+
+import os
+import signal
+import subprocess
+import sys
+import platform
+
+
+def windows_impl(search_pattern):
+
+ cmd = "wmic process get ProcessId,CommandLine"
+ output = subprocess.check_output(cmd, shell=True, universal_newlines=True)
+ lines = output.split("\n")
+
+ PIDs = []
+
+ for line in lines:
+ if "clang" in line and "kill_process" not in line and "llvm-lit" not in line:
+
+ # creates something like ['..\\llvm-project\\build\\bin\\clang.exe -cc1modbuildd mbd-launch -v', '16260']
+ parts = line.rsplit(None, 1)
+
+ if len(parts) == 2:
+ command_line, pid = parts
+ if search_pattern in command_line:
+ PIDs.append(int(pid))
+
+ return PIDs
+
+
+def unix_impl(search_pattern):
+
+ ps_output = subprocess.check_output(["ps", "aux"], universal_newlines=True)
+ ps_lines = ps_output.split("\n")
+
+ PIDs = []
+
+ for line in ps_lines:
+ if "clang" in line and "kill_process" not in line and "llvm-lit" not in line:
+ ps_line_components = line.split()
+
+ # output of ps aux
+ # 0 1 2 3 4 5 6 7 8 9 10
+ # USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
+
+ # skip line without COMMAND
+ if len(ps_line_components) < 11:
+ continue
+
+ command_line = " ".join(ps_line_components[10:])
+ if search_pattern in command_line:
+ PIDs.append(int(ps_line_components[1]))
+
+ return PIDs
+
+
+def main():
+ if len(sys.argv) == 1:
+ sys.stderr.write("Error: no search pattern provided\n")
+ sys.exit(1)
+
+ search_pattern = sys.argv[1]
+ sys.stdout.write(
+ f"Searching for process with pattern '{search_pattern}' in command line\n"
+ )
+
+ system_name = platform.system()
+
+ PIDs = []
+ if system_name == "Windows":
+ PIDs = windows_impl(search_pattern)
+ else:
+ PIDs = unix_impl(search_pattern)
+
+ if len(PIDs) == 0:
+ sys.stdout.write("No processes matching search pattern were found\n")
+ return
+
+ sys.stdout.write(f"Killing PIDs {PIDs}\n")
+ for PID in PIDs:
+ os.kill(PID, signal.SIGTERM)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/llvm/include/llvm/Support/Program.h b/llvm/include/llvm/Support/Program.h
index 4c1133e44a21c9..a6a897704f2081 100644
--- a/llvm/include/llvm/Support/Program.h
+++ b/llvm/include/llvm/Support/Program.h
@@ -152,7 +152,8 @@ namespace sys {
unsigned MemoryLimit = 0,
std::string *ErrMsg = nullptr,
bool *ExecutionFailed = nullptr,
- BitVector *AffinityMask = nullptr);
+ BitVector *AffinityMask = nullptr,
+ bool DetachProcess = false);
/// Return true if the given arguments fit within system-specific
/// argument length limits.
diff --git a/llvm/include/llvm/Support/raw_socket_stream.h b/llvm/include/llvm/Support/raw_socket_stream.h
index c219792d82465d..a8d9361cb0a315 100644
--- a/llvm/include/llvm/Support/raw_socket_stream.h
+++ b/llvm/include/llvm/Support/raw_socket_stream.h
@@ -42,8 +42,14 @@ class ListeningSocket {
static Expected<ListeningSocket> createUnix(
StringRef SocketPath,
int MaxBacklog = llvm::hardware_concurrency().compute_thread_count());
- Expected<std::unique_ptr<raw_socket_stream>> accept();
+
+ // The use of a default parameter was choosen over std::optional to more
+ // closely resemble the accept system call. A user should be able to call
+ // ListeningSocket.accept() rather then ListeningSocket.accept(std::nullopt)
+ // if they do not want accept to ever timeout
+ Expected<std::unique_ptr<raw_socket_stream>> accept(timeval TV = {-1, -1});
ListeningSocket(ListeningSocket &&LS);
+ void shutdown();
~ListeningSocket();
};
class raw_socket_stream : public raw_fd_stream {
diff --git a/llvm/lib/Support/Program.cpp b/llvm/lib/Support/Program.cpp
index 1dcd45e2d69e89..642f6e73f32a77 100644
--- a/llvm/lib/Support/Program.cpp
+++ b/llvm/lib/Support/Program.cpp
@@ -27,7 +27,7 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
std::optional<ArrayRef<StringRef>> Env,
ArrayRef<std::optional<StringRef>> Redirects,
unsigned MemoryLimit, std::string *ErrMsg,
- BitVector *AffinityMask);
+ BitVector *AffinityMask, bool DetachProcess);
int sys::ExecuteAndWait(StringRef Program, ArrayRef<StringRef> Args,
std::optional<ArrayRef<StringRef>> Env,
@@ -39,7 +39,7 @@ int sys::ExecuteAndWait(StringRef Program, ArrayRef<StringRef> Args,
assert(Redirects.empty() || Redirects.size() == 3);
ProcessInfo PI;
if (Execute(PI, Program, Args, Env, Redirects, MemoryLimit, ErrMsg,
- AffinityMask)) {
+ AffinityMask, /*DetachProcess*/ false)) {
if (ExecutionFailed)
*ExecutionFailed = false;
ProcessInfo Result = Wait(
@@ -58,13 +58,14 @@ ProcessInfo sys::ExecuteNoWait(StringRef Program, ArrayRef<StringRef> Args,
std::optional<ArrayRef<StringRef>> Env,
ArrayRef<std::optional<StringRef>> Redirects,
unsigned MemoryLimit, std::string *ErrMsg,
- bool *ExecutionFailed, BitVector *AffinityMask) {
+ bool *ExecutionFailed, BitVector *AffinityMask,
+ bool DetachProcess) {
assert(Redirects.empty() || Redirects.size() == 3);
ProcessInfo PI;
if (ExecutionFailed)
*ExecutionFailed = false;
if (!Execute(PI, Program, Args, Env, Redirects, MemoryLimit, ErrMsg,
- AffinityMask))
+ AffinityMask, DetachProcess))
if (ExecutionFailed)
*ExecutionFailed = true;
diff --git a/llvm/lib/Support/Unix/Program.inc b/llvm/lib/Support/Unix/Program.inc
index 5d9757bcc51b3e..56f653a165bfb5 100644
--- a/llvm/lib/Support/Unix/Program.inc
+++ b/llvm/lib/Support/Unix/Program.inc
@@ -173,10 +173,11 @@ toNullTerminatedCStringArray(ArrayRef<StringRef> Strings, StringSaver &Saver) {
}
static bool Execute(ProcessInfo &PI, StringRef Program,
- ArrayRef<StringRef> Args, std::optional<ArrayRef<StringRef>> Env,
+ ArrayRef<StringRef> Args,
+ std::optional<ArrayRef<StringRef>> Env,
ArrayRef<std::optional<StringRef>> Redirects,
unsigned MemoryLimit, std::string *ErrMsg,
- BitVector *AffinityMask) {
+ BitVector *AffinityMask, bool DetachProcess) {
if (!llvm::sys::fs::exists(Program)) {
if (ErrMsg)
*ErrMsg = std::string("Executable \"") + Program.str() +
@@ -202,7 +203,8 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
// If this OS has posix_spawn and there is no memory limit being implied, use
// posix_spawn. It is more efficient than fork/exec.
#ifdef HAVE_POSIX_SPAWN
- if (MemoryLimit == 0) {
+ // Cannot use posix_spawn if you would like to detach the process
+ if (MemoryLimit == 0 && !DetachProcess) {
posix_spawn_file_actions_t FileActionsStore;
posix_spawn_file_actions_t *FileActions = nullptr;
@@ -270,7 +272,7 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
return true;
}
-#endif
+#endif // HAVE_POSIX_SPAWN
// Create a child process.
int child = fork();
@@ -307,6 +309,14 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
}
}
+ if (DetachProcess) {
+ // Detach from controlling terminal
+ if (setsid() == -1) {
+ MakeErrMsg(ErrMsg, "Couldn't detach process, setsid() failed");
+ return false;
+ }
+ }
+
// Set memory limits
if (MemoryLimit != 0) {
SetMemoryLimits(MemoryLimit);
diff --git a/llvm/lib/Support/Windows/Program.inc b/llvm/lib/Support/Windows/Program.inc
index 0de9d3f7564481..d98d55f317a35a 100644
--- a/llvm/lib/Support/Windows/Program.inc
+++ b/llvm/lib/Support/Windows/Program.inc
@@ -172,10 +172,11 @@ static HANDLE RedirectIO(std::optional<StringRef> Path, int fd,
} // namespace llvm
static bool Execute(ProcessInfo &PI, StringRef Program,
- ArrayRef<StringRef> Args, std::optional<ArrayRef<StringRef>> Env,
+ ArrayRef<StringRef> Args,
+ std::optional<ArrayRef<StringRef>> Env,
ArrayRef<std::optional<StringRef>> Redirects,
unsigned MemoryLimit, std::string *ErrMsg,
- BitVector *AffinityMask) {
+ BitVector *AffinityMask, bool DetachProcess) {
if (!sys::fs::can_execute(Program)) {
if (ErrMsg)
*ErrMsg = "program not executable";
@@ -284,6 +285,8 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
unsigned CreateFlags = CREATE_UNICODE_ENVIRONMENT;
if (AffinityMask)
CreateFlags |= CREATE_SUSPENDED;
+ if (DetachProcess)
+ CreateFlags |= DETACHED_PROCESS;
std::vector<wchar_t> CommandUtf16(Command.size() + 1, 0);
std::copy(Command.begin(), Command.end(), CommandUtf16.begin());
diff --git a/llvm/lib/Support/raw_socket_stream.cpp b/llvm/lib/Support/raw_socket_stream.cpp
index a65865bcede12d..efc8f813d05faf 100644
--- a/llvm/lib/Support/raw_socket_stream.cpp
+++ b/llvm/lib/Support/raw_socket_stream.cpp
@@ -103,26 +103,60 @@ Expected<ListeningSocket> ListeningSocket::createUnix(StringRef SocketPath,
return ListeningSocket{UnixSocket, SocketPath};
}
-Expected<std::unique_ptr<raw_socket_stream>> ListeningSocket::accept() {
+Expected<std::unique_ptr<raw_socket_stream>>
+ListeningSocket::accept(timeval TV) {
+
+ int SelectStatus;
int AcceptFD;
+
#ifdef _WIN32
SOCKET WinServerSock = _get_osfhandle(FD);
- SOCKET WinAcceptSock = ::accept(WinServerSock, NULL, NULL);
- AcceptFD = _open_osfhandle(WinAcceptSock, 0);
+#endif
+
+ fd_set Readfds;
+ if (TV.tv_sec != -1 && TV.tv_usec != -1) {
+ FD_ZERO(&Readfds);
+#ifdef _WIN32
+ FD_SET(WinServerSock, &Readfds);
#else
- AcceptFD = ::accept(FD, NULL, NULL);
-#endif //_WIN32
+ FD_SET(FD, &Readfds);
+#endif
+ SelectStatus = ::select(FD + 1, &Readfds, NULL, NULL, &TV);
+ } else
+ SelectStatus = ::select(FD + 1, &Readfds, NULL, NULL, NULL);
+
+ if (SelectStatus == -1)
+ return llvm::make_error<StringError>(getLastSocketErrorCode(),
+ "Select failed");
+ else if (SelectStatus) {
+#ifdef _WIN32
+ SOCKET WinAcceptSock = ::accept(WinServerSock, NULL, NULL);
+ AcceptFD = _open_osfhandle(WinAcceptSock, 0);
+#else
+ AcceptFD = ::accept(FD, NULL, NULL);
+#endif
+ } else
+ return llvm::make_error<StringError>(
+ std::make_error_code(std::errc::timed_out), "Accept timed out");
+
if (AcceptFD == -1)
return llvm::make_error<StringError>(getLastSocketErrorCode(),
"Accept failed");
+
return std::make_unique<raw_socket_stream>(AcceptFD);
}
-ListeningSocket::~ListeningSocket() {
+void ListeningSocket::shutdown() {
if (FD == -1)
return;
::close(FD);
- unlink(SocketPath.c_str());
+ ::unlink(SocketPath.c_str());
+
+ FD = -1;
+}
+
+ListeningSocket::~ListeningSocket() {
+ shutdown();
}
static Expected<int> GetSocketFD(StringRef SocketPath) {
More information about the cfe-commits
mailing list