[clang] Added partial support for compiling C++20 modules and header-units without scanning. (PR #147682)
Hassan Sajjad via cfe-commits
cfe-commits at lists.llvm.org
Thu Sep 24 01:19:09 PDT 2026
https://github.com/HassanSajjad-302 updated https://github.com/llvm/llvm-project/pull/147682
>From fcc02241c15c62b0926e68879f3462ca2969b5c6 Mon Sep 17 00:00:00 2001
From: Hassan Sajjad <hassan.sajjad069 at gmail.com>
Date: Wed, 23 Sep 2026 18:19:46 +0500
Subject: [PATCH] P2978 Implemented
---
clang/CMakeLists.txt | 7 +
.../include/clang/Frontend/CompilerInstance.h | 2 +
.../clang/IPC2978/.clang-format-ignore | 1 +
clang/include/clang/IPC2978/IPCManagerBS.hpp | 21 +
.../clang/IPC2978/IPCManagerCompiler.hpp | 89 ++
clang/include/clang/IPC2978/Manager.hpp | 84 ++
clang/include/clang/IPC2978/Messages.hpp | 88 ++
clang/include/clang/IPC2978/Result.hpp | 122 +++
clang/include/clang/Lex/HeaderSearchOptions.h | 6 +
clang/include/clang/Lex/ModuleLoader.h | 10 +
clang/include/clang/Options/Options.td | 8 +
clang/lib/CMakeLists.txt | 1 +
clang/lib/Driver/ToolChains/Clang.cpp | 9 +
clang/lib/Frontend/CompilerInstance.cpp | 85 +-
clang/lib/Frontend/CompilerInvocation.cpp | 10 +
clang/lib/IPC2978/.clang-format-ignore | 1 +
clang/lib/IPC2978/CMakeLists.txt | 6 +
clang/lib/IPC2978/IPCManagerBS.cpp | 57 ++
clang/lib/IPC2978/IPCManagerCompiler.cpp | 464 ++++++++++
clang/lib/IPC2978/Manager.cpp | 283 ++++++
clang/lib/IPC2978/setup.py | 24 +
clang/lib/Lex/CMakeLists.txt | 1 +
clang/lib/Lex/PPDirectives.cpp | 58 +-
clang/lib/Parse/Parser.cpp | 12 +-
clang/lib/Serialization/ASTReader.cpp | 25 +-
clang/lib/Serialization/ModuleManager.cpp | 25 +-
clang/unittests/CMakeLists.txt | 3 +
clang/unittests/IPC2978/.clang-format-ignore | 1 +
clang/unittests/IPC2978/CMakeLists.txt | 12 +
clang/unittests/IPC2978/IPC2978Test.cpp | 819 ++++++++++++++++++
clang/unittests/IPC2978/TestBuildSystem.hpp | 61 ++
clang/unittests/IPC2978/TestProcess.hpp | 317 +++++++
32 files changed, 2691 insertions(+), 21 deletions(-)
create mode 100644 clang/include/clang/IPC2978/.clang-format-ignore
create mode 100644 clang/include/clang/IPC2978/IPCManagerBS.hpp
create mode 100644 clang/include/clang/IPC2978/IPCManagerCompiler.hpp
create mode 100644 clang/include/clang/IPC2978/Manager.hpp
create mode 100644 clang/include/clang/IPC2978/Messages.hpp
create mode 100644 clang/include/clang/IPC2978/Result.hpp
create mode 100644 clang/lib/IPC2978/.clang-format-ignore
create mode 100644 clang/lib/IPC2978/CMakeLists.txt
create mode 100644 clang/lib/IPC2978/IPCManagerBS.cpp
create mode 100644 clang/lib/IPC2978/IPCManagerCompiler.cpp
create mode 100644 clang/lib/IPC2978/Manager.cpp
create mode 100755 clang/lib/IPC2978/setup.py
create mode 100644 clang/unittests/IPC2978/.clang-format-ignore
create mode 100644 clang/unittests/IPC2978/CMakeLists.txt
create mode 100644 clang/unittests/IPC2978/IPC2978Test.cpp
create mode 100644 clang/unittests/IPC2978/TestBuildSystem.hpp
create mode 100644 clang/unittests/IPC2978/TestProcess.hpp
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index a1bdd56599e86c..7ff8b174f30d6d 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -453,6 +453,13 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
PATTERN "config.h" EXCLUDE
)
+ install(DIRECTORY include/clang/IPC2978
+ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/clang"
+ COMPONENT clang-headers
+ FILES_MATCHING
+ PATTERN "*.hpp"
+ )
+
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/clang
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
COMPONENT clang-headers
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index 21b83dfefa8426..c2da08b839cf6d 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -985,6 +985,8 @@ class CompilerInstance : public ModuleLoader {
GenModuleActionWrapperFunc getGenModuleActionWrapper() const {
return GenModuleActionWrapper;
}
+ Module *loadIPCReceivedHeaderUnit(StringRef FileName,
+ SourceLocation ImportLoc) override;
void addDependencyCollector(std::shared_ptr<DependencyCollector> Listener) {
DependencyCollectors.push_back(std::move(Listener));
diff --git a/clang/include/clang/IPC2978/.clang-format-ignore b/clang/include/clang/IPC2978/.clang-format-ignore
new file mode 100644
index 00000000000000..f59ec20aabf584
--- /dev/null
+++ b/clang/include/clang/IPC2978/.clang-format-ignore
@@ -0,0 +1 @@
+*
\ No newline at end of file
diff --git a/clang/include/clang/IPC2978/IPCManagerBS.hpp b/clang/include/clang/IPC2978/IPCManagerBS.hpp
new file mode 100644
index 00000000000000..2b2b396550c447
--- /dev/null
+++ b/clang/include/clang/IPC2978/IPCManagerBS.hpp
@@ -0,0 +1,21 @@
+
+#ifndef IPC_MANAGER_BS_HPP
+#define IPC_MANAGER_BS_HPP
+
+#include "clang/IPC2978/Manager.hpp"
+#include "clang/IPC2978/Messages.hpp"
+
+namespace P2978
+{
+
+// Parses dependency requests received by the build system.
+class IPCManagerBS
+{
+ public:
+ // Parse one payload after the caller removes diagnostics, payload size, and delimiter.
+ // ctbBuffer must be aligned for CTBModule/CTBNonModule. Parsed string views borrow
+ // serverReadString, whose bytes must remain alive until the request is consumed.
+ static Result<void> receiveMessage(char (&ctbBuffer)[320], CTB &messageType, std::string_view serverReadString);
+};
+} // namespace P2978
+#endif // IPC_MANAGER_BS_HPP
diff --git a/clang/include/clang/IPC2978/IPCManagerCompiler.hpp b/clang/include/clang/IPC2978/IPCManagerCompiler.hpp
new file mode 100644
index 00000000000000..cebc4db5095995
--- /dev/null
+++ b/clang/include/clang/IPC2978/IPCManagerCompiler.hpp
@@ -0,0 +1,89 @@
+
+#ifndef IPC_MANAGER_COMPILER_HPP
+#define IPC_MANAGER_COMPILER_HPP
+
+#include "clang/IPC2978/Manager.hpp"
+
+#include <memory>
+#include <unordered_map>
+
+struct CompilerTest;
+struct BuildSystemTest;
+namespace P2978
+{
+
+enum class FileType : uint8_t
+{
+ MODULE,
+ HEADER_UNIT,
+ HEADER_FILE
+};
+
+struct Response
+{
+ // Borrows the manager's message storage; keep the manager alive while using this path.
+ std::string_view filePath;
+ // Empty for a textual header. BMI views remain mapped until the compiler process exits.
+ std::string_view bmiContents;
+ FileType type;
+ bool isSystem;
+};
+
+// One dependency session per compiler process. Each request may populate several cached responses.
+class IPCManagerCompiler : Manager
+{
+ friend struct ::CompilerTest;
+ friend struct ::BuildSystemTest;
+
+ Result<std::string_view> readInternal(char (&buffer)[4096]) const;
+ // Send already-framed bytes through the compiler's pipe.
+ Result<void> writeInternal(std::string_view buffer) const;
+
+ // Decode a BMI path, reusing its mapped contents if an earlier response supplied it.
+ Result<Response> readBMIResponse(std::string_view message, uint64_t &bytesRead, FileType type,
+ bool isSystem = true);
+ Result<void> readLogicalNames(std::string_view message, uint64_t &bytesRead, const Response &response);
+
+ // Resolve a cache miss and retain the requested entry and all accompanying dependencies.
+ [[nodiscard]] Result<void> receiveBTCModule(const CTBModule &moduleName);
+ [[nodiscard]] Result<void> receiveBTCNonModule(const CTBNonModule &nonModule);
+
+ // Logical-name keys and response paths borrow the retained message or mock-file storage below.
+ std::unordered_map<std::string_view, Response> responses;
+
+ // Paths borrow retained message or mock-file storage, just like responses.
+ // Each path is mapped once per session. Erasing this cache would not unmap its views.
+ std::unordered_map<std::string_view, std::string_view> bmiContentsByPath;
+
+ // Initialized once so loading another mock cannot invalidate existing string views.
+ std::string scanCacheFileData;
+ bool isMocking = false;
+
+ // Successful mappings belong to the process, not to the manager or the returned string_view.
+ static Result<std::string_view> mapBMIFile(std::string_view filePath);
+ // filePath must borrow retained message or mock-file storage.
+ // Map the completed file only on the first request for its path.
+ Result<std::string_view> loadBMIContents(std::string_view filePath);
+
+ // Separate allocations keep string addresses stable as subsequent messages arrive.
+ mutable std::vector<std::unique_ptr<std::string>> allocations;
+
+ public:
+ // Retained for Clang's command-line reproduction after successful mock initialization.
+ std::string mockFilePath;
+
+ // Load mock dependencies once on a fresh manager. A failed attempt also disables live IPC.
+ Result<void> readEntriesFromFile(std::string_view filePath);
+
+ // Look up a BMI already received from the build system; this does not map files or send requests.
+ // Use the same normalized path supplied in the response (lowercase on Windows).
+ [[nodiscard]] Result<std::string_view> findBMIContents(std::string_view filePath) const;
+
+ // A textual-header request may resolve to a header unit for include translation. Other kinds must match.
+ // Cache misses use one request/reply exchange; mock sessions report a missing entry instead.
+ [[nodiscard]] Result<Response> findResponse(std::string_view logicalName, FileType type);
+};
+
+inline IPCManagerCompiler *managerCompiler;
+} // namespace P2978
+#endif // IPC_MANAGER_COMPILER_HPP
diff --git a/clang/include/clang/IPC2978/Manager.hpp b/clang/include/clang/IPC2978/Manager.hpp
new file mode 100644
index 00000000000000..a1a44789b4020c
--- /dev/null
+++ b/clang/include/clang/IPC2978/Manager.hpp
@@ -0,0 +1,84 @@
+
+#ifndef MANAGER_HPP
+#define MANAGER_HPP
+
+#include "clang/IPC2978/Messages.hpp"
+#include "clang/IPC2978/Result.hpp"
+
+#include <new>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace P2978
+{
+
+// A 32-byte marker terminates each pipe message. Compiler requests also put a
+// uint32_t payload size immediately before it, separating requests from diagnostics.
+inline const char *delimiter = "DELIMITER"
+ "\x5A\xA5\x5A\xA5\x5A\xA5\x5A\xA5\x5A\xA5\x5A\xA5\x5A\xA5"
+ "DELIMITER";
+
+enum class ErrorCategory : uint8_t
+{
+ NONE,
+ PARSING_ERROR,
+ READ_FILE_ZERO_BYTES_READ,
+ UNKNOWN_CTB_TYPE,
+};
+
+// Describe the current errno on Unix or GetLastError() on Windows.
+std::string getErrorString();
+std::string getErrorString(uint64_t bytesRead_, uint64_t bytesProcessed_);
+std::string getErrorString(ErrorCategory errorCategory_);
+
+// Shared wire encoding and pipe I/O helpers; endpoints do not use polymorphism.
+class Manager
+{
+ public:
+ // Complete partial writes; the caller owns the descriptor or handle.
+#ifndef _WIN32
+ static Result<void> writeAll(const int fd, const char *buffer, const uint64_t count);
+#else
+ static Result<void> writeAll(void *handle, std::string_view buffer);
+#endif
+
+ static std::string getBufferWithType(CTB type);
+ // Wire lengths and counts use native-endian uint32_t values.
+ static void writeUInt32(std::string &buffer, uint32_t value);
+ static void writeString(std::string &buffer, const std::string_view &str);
+ // Paths include a trailing NUL for OS calls; the encoded length excludes it.
+ static void writePath(std::string &buffer, const std::string_view &str);
+ static void writeModuleDep(std::string &buffer, const ModuleDep &dep);
+ static void writeHuDep(std::string &buffer, const HuDep &dep);
+ static void writeHeaderFile(std::string &buffer, const HeaderFile &dep);
+ static void writeVectorOfStrings(std::string &buffer, const std::vector<std::string_view> &strs);
+ static void writeVectorOfModuleDep(std::string &buffer, const std::vector<ModuleDep> &deps);
+ static void writeVectorOfHuDeps(std::string &buffer, const std::vector<HuDep> &deps);
+ static void writeVectorOfHeaderFiles(std::string &buffer, const std::vector<HeaderFile> &headerFiles);
+
+ // Parsing offsets use uint64_t independently of the wire field widths.
+ static Result<bool> readBool(std::string_view message, uint64_t &bytesRead);
+ static Result<uint8_t> readUInt8(std::string_view message, uint64_t &bytesRead);
+ static Result<uint32_t> readUInt32(std::string_view message, uint64_t &bytesRead);
+ static Result<std::string_view> readString(std::string_view message, uint64_t &bytesRead);
+
+ // Returned string/path views borrow message. A path's view excludes its NUL,
+ // but the parser consumes and validates that byte before returning.
+ static Result<std::string_view> readPath(std::string_view message, uint64_t &bytesRead);
+};
+
+template <typename T, typename... Args> constexpr T *construct_at(T *p, Args &&...args)
+{
+ return ::new (static_cast<void *>(p)) T(std::forward<Args>(args)...);
+}
+
+template <typename T> T &getInitializedObjectFromBuffer(char (&buffer)[320])
+{
+ T &t = reinterpret_cast<T &>(buffer);
+ construct_at(&t);
+ return t;
+}
+
+} // namespace P2978
+#endif // MANAGER_HPP
diff --git a/clang/include/clang/IPC2978/Messages.hpp b/clang/include/clang/IPC2978/Messages.hpp
new file mode 100644
index 00000000000000..b70d73fce1bbbd
--- /dev/null
+++ b/clang/include/clang/IPC2978/Messages.hpp
@@ -0,0 +1,88 @@
+#ifndef MESSAGES_HPP
+#define MESSAGES_HPP
+
+#include <cstdint>
+#include <string_view>
+#include <vector>
+
+namespace P2978
+{
+// CTB: compiler to build system. BTC: build system to compiler.
+// Strings use a uint32_t byte length followed by their bytes. Paths add a NUL
+// outside that length. Vectors use a uint32_t element count followed by elements.
+// Integers are native-endian; bools and request types occupy one byte.
+// Fields are serialized in declaration order, except for BTCNonModule's optional tail.
+
+// A request begins with this tag. Each compiler waits for its response before sending another request.
+enum class CTB : uint8_t
+{
+ MODULE = 0,
+ NON_MODULE = 1,
+};
+
+// Resolve a named module or module partition.
+struct CTBModule
+{
+ std::string_view moduleName;
+};
+
+// Resolve an include or a header-unit import. An include may resolve to a header unit.
+struct CTBNonModule
+{
+ bool isHeaderUnit = false;
+ std::string_view logicalName;
+};
+
+// Responses need no type tag: the outstanding request determines which layout to read.
+// A BMI must be complete before publication and remain available to its consumers.
+struct ModuleDep
+{
+ bool isHeaderUnit = false;
+ std::string_view filePath;
+ // Classify the dependency as system input for compiler diagnostics.
+ bool isSystem = true;
+ // A module has one name. A composed header unit may provide several include-name aliases.
+ std::vector<std::string_view> logicalNames;
+};
+
+// Reply to CTBModule, including dependencies needed to load the requested BMI.
+struct BTCModule
+{
+ std::string_view filePath;
+ bool isSystem = true;
+ // Omit dependencies already supplied to this compiler; their cached responses remain valid.
+ std::vector<ModuleDep> modDeps;
+};
+
+struct HuDep
+{
+ std::string_view filePath;
+ // Classify the header unit as system input for compiler diagnostics.
+ bool isSystem = true;
+ // Include names that resolve to this same header-unit BMI.
+ std::vector<std::string_view> logicalNames;
+};
+
+struct HeaderFile
+{
+ std::string_view logicalName;
+ std::string_view filePath;
+ bool isSystem = true;
+};
+
+// Reply to CTBNonModule. Batch known headers and BMI dependencies to avoid further round trips.
+struct BTCNonModule
+{
+ bool isHeaderUnit = false;
+ bool isSystem = true;
+ // Additional textual headers to cache, for example the headers composing the unit being built.
+ std::vector<HeaderFile> headerFiles;
+ std::string_view filePath;
+ // The remaining fields are sent only for a header-unit response. Otherwise filePath names a textual header.
+ // Additional aliases for the requested BMI; the request's logical name is cached implicitly.
+ std::vector<std::string_view> logicalNames;
+ std::vector<HuDep> huDeps;
+};
+
+} // namespace P2978
+#endif // MESSAGES_HPP
diff --git a/clang/include/clang/IPC2978/Result.hpp b/clang/include/clang/IPC2978/Result.hpp
new file mode 100644
index 00000000000000..5e1ff3c29ca1b9
--- /dev/null
+++ b/clang/include/clang/IPC2978/Result.hpp
@@ -0,0 +1,122 @@
+#ifndef IPC2978_RESULT_HPP
+#define IPC2978_RESULT_HPP
+
+#include <cassert>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+namespace P2978
+{
+
+// Explicitly distinguish errors from successful string values, including empty strings.
+struct Error
+{
+ std::string message;
+};
+
+// C++17 result with an owned error message. Check success before dereferencing,
+// and failure before calling error(); invalid access is a programming error.
+// The variant manages value lifetimes and copy/move support, including move-only T.
+template <typename T> class [[nodiscard]] Result
+{
+ std::variant<T, Error> storage;
+
+ public:
+ template <typename U = T, std::enable_if_t<std::is_copy_constructible_v<U>, int> = 0>
+ Result(const T &value) : storage(std::in_place_index<0>, value)
+ {
+ }
+ template <typename U = T, std::enable_if_t<std::is_move_constructible_v<U>, int> = 0>
+ Result(T &&value) : storage(std::in_place_index<0>, std::move(value))
+ {
+ }
+ Result(Error error) : storage(std::in_place_index<1>, std::move(error))
+ {
+ }
+
+ explicit operator bool() const noexcept
+ {
+ return storage.index() == 0;
+ }
+
+ T *operator->() noexcept
+ {
+ auto *value = std::get_if<0>(&storage);
+ assert(value);
+ return value;
+ }
+ const T *operator->() const noexcept
+ {
+ const auto *value = std::get_if<0>(&storage);
+ assert(value);
+ return value;
+ }
+ T &operator*() & noexcept
+ {
+ return *operator->();
+ }
+ const T &operator*() const & noexcept
+ {
+ return *operator->();
+ }
+ T &&operator*() && noexcept
+ {
+ return std::move(*operator->());
+ }
+
+ std::string &error() & noexcept
+ {
+ auto *error = std::get_if<1>(&storage);
+ assert(error);
+ return error->message;
+ }
+ const std::string &error() const & noexcept
+ {
+ const auto *error = std::get_if<1>(&storage);
+ assert(error);
+ return error->message;
+ }
+ std::string &&error() && noexcept
+ {
+ return std::move(this->error());
+ }
+};
+
+// A void result succeeds with {}; even an empty Error represents failure.
+template <> class [[nodiscard]] Result<void>
+{
+ std::variant<std::monostate, Error> storage;
+
+ public:
+ Result() = default;
+ Result(Error error) : storage(std::in_place_index<1>, std::move(error))
+ {
+ }
+
+ explicit operator bool() const noexcept
+ {
+ return storage.index() == 0;
+ }
+
+ std::string &error() & noexcept
+ {
+ auto *error = std::get_if<1>(&storage);
+ assert(error);
+ return error->message;
+ }
+ const std::string &error() const & noexcept
+ {
+ const auto *error = std::get_if<1>(&storage);
+ assert(error);
+ return error->message;
+ }
+ std::string &&error() && noexcept
+ {
+ return std::move(this->error());
+ }
+};
+
+} // namespace P2978
+#endif // IPC2978_RESULT_HPP
diff --git a/clang/include/clang/Lex/HeaderSearchOptions.h b/clang/include/clang/Lex/HeaderSearchOptions.h
index a521f38e28233a..ec286df0018be8 100644
--- a/clang/include/clang/Lex/HeaderSearchOptions.h
+++ b/clang/include/clang/Lex/HeaderSearchOptions.h
@@ -119,6 +119,12 @@ class HeaderSearchOptions {
/// The directories used to load prebuilt module files.
std::vector<std::string> PrebuiltModulePaths;
+ /// Resolve dependencies through the build-system IPC session.
+ bool UseIPC = false;
+
+ /// Use dependencies from this mock file instead of live IPC when non-empty.
+ std::string IPCMockFile;
+
/// The module/pch container format.
std::string ModuleFormat;
diff --git a/clang/include/clang/Lex/ModuleLoader.h b/clang/include/clang/Lex/ModuleLoader.h
index 042a5ab1f4a57e..cffbe51d63f321 100644
--- a/clang/include/clang/Lex/ModuleLoader.h
+++ b/clang/include/clang/Lex/ModuleLoader.h
@@ -161,6 +161,11 @@ class ModuleLoader {
SourceLocation TriggerLoc) = 0;
static std::string getFlatNameFromPath(ModuleIdPath Path);
+ /// Load a C++20 header unit received over IPC, reusing an already loaded BMI.
+ /// Returns nullptr if the file cannot be loaded as a header unit.
+ virtual Module *loadIPCReceivedHeaderUnit(StringRef FileName,
+ SourceLocation ImportLoc) = 0;
+
bool HadFatalFailure = false;
};
@@ -187,6 +192,11 @@ class TrivialModuleLoader : public ModuleLoader {
SourceLocation TriggerLoc) override {
return false;
}
+
+ Module *loadIPCReceivedHeaderUnit(StringRef FileName,
+ SourceLocation ImportLoc) override {
+ return nullptr;
+ }
};
} // namespace clang
diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index d4bf48040029f5..4778a302c66de4 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -10130,3 +10130,11 @@ def wasm_opt : Flag<["--"], "wasm-opt">,
Group<m_Group>,
HelpText<"Enable the wasm-opt optimizer (default)">,
MarshallingInfoNegativeFlag<LangOpts<"NoWasmOpt">>;
+
+def use_ipc : Flag<["-", "/"], "useIPC">,
+ Visibility<[ClangOption, CC1Option, CLOption]>,
+ HelpText<"Enable IPC approach">;
+
+def use_ipc_file : Joined<["-", "/"], "useIPC=">,
+ Visibility<[ClangOption, CC1Option, CLOption]>,
+ HelpText<"Enable IPC approach with mock file">;
\ No newline at end of file
diff --git a/clang/lib/CMakeLists.txt b/clang/lib/CMakeLists.txt
index 22d260e55b2711..967e7b8201f3e6 100644
--- a/clang/lib/CMakeLists.txt
+++ b/clang/lib/CMakeLists.txt
@@ -25,6 +25,7 @@ add_subdirectory(Index)
add_subdirectory(IndexSerialization)
add_subdirectory(InstallAPI)
add_subdirectory(ScalableStaticAnalysis)
+add_subdirectory(IPC2978)
add_subdirectory(StaticAnalyzer)
add_subdirectory(Format)
if(CLANG_INCLUDE_TESTS)
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 6636a5fd6e6551..20b4fd58c7d997 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -4311,6 +4311,15 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
Args.ClaimAllArgs(options::OPT_fmodules_reduced_bmi);
Args.ClaimAllArgs(options::OPT_fno_modules_reduced_bmi);
+ if (const Arg *A = Args.getLastArg(options::OPT_use_ipc_file)) {
+ CmdArgs.push_back(
+ Args.MakeArgString(llvm::Twine("-useIPC=") + A->getValue()));
+ A->claim();
+ } else if (Args.hasArg(options::OPT_use_ipc)) {
+ CmdArgs.push_back("-useIPC");
+ Args.claimAllArgs(options::OPT_use_ipc);
+ }
+
// We need to include the case the input file is a module file here.
// Since the default compilation model for C++ module interface unit will
// create temporary module file and compile the temporary module file
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 87abcd38c1a927..9374802d1bbeb7 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -32,6 +32,7 @@
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/Frontend/VerifyDiagnosticConsumer.h"
+#include "clang/IPC2978/IPCManagerCompiler.hpp"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PreprocessorOptions.h"
@@ -58,6 +59,7 @@
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SmallVectorMemoryBuffer.h"
#include "llvm/Support/Threading.h"
@@ -995,6 +997,24 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
llvm::TimeTraceScope TimeScope("ExecuteCompiler");
+ // Argument parsing (including round-tripping) must not start an IPC session.
+ // Keep one dependency cache for the entire action, including all its inputs.
+ std::unique_ptr<P2978::IPCManagerCompiler> IPCManager;
+ if (getHeaderSearchOpts().UseIPC) {
+ IPCManager = std::make_unique<P2978::IPCManagerCompiler>();
+ const std::string &MockFile = getHeaderSearchOpts().IPCMockFile;
+ if (!MockFile.empty()) {
+ if (const auto Result = IPCManager->readEntriesFromFile(MockFile);
+ !Result) {
+ getDiagnostics().Report(getDiagnostics().getCustomDiagID(
+ DiagnosticsEngine::Error, "could not load IPC mock file: %0"))
+ << Result.error();
+ return false;
+ }
+ }
+ }
+ llvm::SaveAndRestore IPCSession(P2978::managerCompiler, IPCManager.get());
+
PrepareForExecution();
// Mark this point as the bottom of the stack if we don't have somewhere
@@ -1929,9 +1949,27 @@ ModuleLoadResult CompilerInstance::findOrCompileModuleAndReadAST(
// Select the source and filename for loading the named module.
ModuleFileName ModuleFilename;
- ModuleSource Source =
- selectModuleSource(M, ModuleName, ModuleFilename, BuiltModules, HS);
SourceLocation ModuleNameLoc = ModuleNameRange.getBegin();
+ ModuleSource Source;
+
+ if (P2978::managerCompiler) {
+ const auto &Result = P2978::managerCompiler->findResponse(
+ ModuleName, P2978::FileType::MODULE);
+ if (!Result) {
+ getDiagnostics().Report(ModuleNameLoc,
+ getDiagnostics().getCustomDiagID(
+ DiagnosticsEngine::Error,
+ "could not resolve IPC dependency '%0': %1"))
+ << ModuleName << Result.error();
+ return ModuleLoadResult();
+ }
+ ModuleFilename = ModuleFileName::makeExplicit(Result->filePath);
+ Source = MS_PrebuiltModulePath;
+ } else {
+ Source =
+ selectModuleSource(M, ModuleName, ModuleFilename, BuiltModules, HS);
+ }
+
if (Source == MS_ModuleNotFound) {
// We can't find a module, error out here.
getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_found)
@@ -2456,6 +2494,49 @@ CompilerInstance::lookupMissingImports(StringRef Name,
return false;
}
+
+Module *
+CompilerInstance::loadIPCReceivedHeaderUnit(const StringRef FileName,
+ const SourceLocation ImportLoc) {
+ if (!getASTReader()) {
+ createASTReader();
+ }
+
+ serialization::ModuleFile *ModuleFile =
+ getASTReader()->getModuleManager().lookupByFileName(
+ ModuleFileName::makeExplicit(FileName));
+ if (!ModuleFile &&
+ !loadModuleFile(ModuleFileName::makeExplicit(FileName), ModuleFile)) {
+ return nullptr;
+ }
+ // A recoverable configuration mismatch can return success without a module.
+ if (!ModuleFile) {
+ getDiagnostics().Report(
+ ImportLoc,
+ getDiagnostics().getCustomDiagID(DiagnosticsEngine::Error,
+ "could not load IPC header unit '%0'"))
+ << FileName;
+ return nullptr;
+ }
+
+ Module *Mod = PP->getHeaderSearchInfo().getModuleMap().findModule(
+ ModuleFile->ModuleName);
+ if (!Mod || !Mod->isHeaderUnit()) {
+ getDiagnostics().Report(
+ ImportLoc, getDiagnostics().getCustomDiagID(
+ DiagnosticsEngine::Error,
+ "IPC dependency '%0' was not built as a header unit"))
+ << FileName;
+ return nullptr;
+ }
+
+ ModuleFile->Kind = serialization::MK_PrebuiltModule;
+ // Use normal import visibility rather than exposing every BMI dependency.
+ // Preprocessor and Sema handle their visibility at the import token.
+ makeModuleVisible(Mod, Module::AllVisible, ImportLoc);
+ return Mod;
+}
+
void CompilerInstance::resetAndLeakSema() { llvm::BuryPointer(takeSema()); }
void CompilerInstance::setExternalSemaSource(
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 649a1a18b4ebe8..9bf06a53f0be0b 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3438,6 +3438,13 @@ static void GenerateHeaderSearchArgs(const HeaderSearchOptions &Opts,
for (const std::string &F : Opts.VFSOverlayFiles)
GenerateArg(Consumer, OPT_ivfsoverlay, F);
+
+ if (Opts.UseIPC) {
+ if (!Opts.IPCMockFile.empty())
+ GenerateArg(Consumer, OPT_use_ipc_file, Opts.IPCMockFile);
+ else
+ GenerateArg(Consumer, OPT_use_ipc);
+ }
}
static bool ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
@@ -3561,6 +3568,9 @@ static bool ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
for (const auto *A : Args.filtered(OPT_ivfsoverlay, OPT_vfsoverlay))
Opts.AddVFSOverlayFile(A->getValue());
+ Opts.UseIPC = Args.hasArg(OPT_use_ipc, OPT_use_ipc_file);
+ Opts.IPCMockFile = Args.getLastArgValue(OPT_use_ipc_file).str();
+
return Diags.getNumErrors() == NumErrorsBefore;
}
diff --git a/clang/lib/IPC2978/.clang-format-ignore b/clang/lib/IPC2978/.clang-format-ignore
new file mode 100644
index 00000000000000..f59ec20aabf584
--- /dev/null
+++ b/clang/lib/IPC2978/.clang-format-ignore
@@ -0,0 +1 @@
+*
\ No newline at end of file
diff --git a/clang/lib/IPC2978/CMakeLists.txt b/clang/lib/IPC2978/CMakeLists.txt
new file mode 100644
index 00000000000000..1ea37f9c9983aa
--- /dev/null
+++ b/clang/lib/IPC2978/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+add_clang_library(clangIPC2978 STATIC
+ Manager.cpp
+ IPCManagerBS.cpp
+ IPCManagerCompiler.cpp
+)
\ No newline at end of file
diff --git a/clang/lib/IPC2978/IPCManagerBS.cpp b/clang/lib/IPC2978/IPCManagerBS.cpp
new file mode 100644
index 00000000000000..64b7a212f43526
--- /dev/null
+++ b/clang/lib/IPC2978/IPCManagerBS.cpp
@@ -0,0 +1,57 @@
+#include "clang/IPC2978/IPCManagerBS.hpp"
+#define TRY_READ_VAL(var, func, ...) \
+ const auto &var##_result = func(__VA_ARGS__); \
+ if (!var##_result) \
+ { \
+ return Error{var##_result.error()}; \
+ } \
+ auto &var = *var##_result;
+
+namespace P2978
+{
+
+Result<void> IPCManagerBS::receiveMessage(char (&ctbBuffer)[320], CTB &messageType,
+ const std::string_view serverReadString)
+{
+ if (serverReadString.empty())
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+
+ uint64_t bytesRead = 1;
+
+ // The nonempty payload starts with a one-byte request type.
+ switch (static_cast<CTB>(serverReadString[0]))
+ {
+
+ case CTB::MODULE: {
+ TRY_READ_VAL(r, Manager::readString, serverReadString, bytesRead);
+
+ messageType = CTB::MODULE;
+ getInitializedObjectFromBuffer<CTBModule>(ctbBuffer).moduleName = r;
+ }
+ break;
+
+ case CTB::NON_MODULE: {
+ TRY_READ_VAL(r, Manager::readBool, serverReadString, bytesRead);
+ TRY_READ_VAL(r2, Manager::readString, serverReadString, bytesRead);
+ messageType = CTB::NON_MODULE;
+ auto &[isHeaderUnit, str] = getInitializedObjectFromBuffer<CTBNonModule>(ctbBuffer);
+ isHeaderUnit = r;
+ str = r2;
+ }
+ break;
+
+ default:
+ return Error{getErrorString(ErrorCategory::UNKNOWN_CTB_TYPE)};
+ }
+
+ if (serverReadString.size() != bytesRead)
+ {
+ return Error{getErrorString(serverReadString.size(), bytesRead)};
+ }
+
+ return {};
+}
+
+} // namespace P2978
diff --git a/clang/lib/IPC2978/IPCManagerCompiler.cpp b/clang/lib/IPC2978/IPCManagerCompiler.cpp
new file mode 100644
index 00000000000000..3f6987d87411d2
--- /dev/null
+++ b/clang/lib/IPC2978/IPCManagerCompiler.cpp
@@ -0,0 +1,464 @@
+
+#include "clang/IPC2978/IPCManagerCompiler.hpp"
+#include "clang/IPC2978/Manager.hpp"
+#include "clang/IPC2978/Messages.hpp"
+
+#include <cctype>
+#include <cerrno>
+#include <cstring>
+#include <fstream>
+#include <iterator>
+#include <limits>
+#include <string>
+#include <utility>
+
+#ifdef _WIN32
+#include <Windows.h>
+#else
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
+
+#define TRY_READ(var, func, ...) \
+ const auto &var = func(__VA_ARGS__); \
+ if (!var) \
+ { \
+ return Error{var.error()}; \
+ }
+
+#define TRY_READ_VAL(var, func, ...) \
+ const auto &var##_result = func(__VA_ARGS__); \
+ if (!var##_result) \
+ { \
+ return Error{var##_result.error()}; \
+ } \
+ auto &var = *var##_result;
+
+namespace P2978
+{
+
+static bool endsWith(const std::string_view str, const std::string_view suffix)
+{
+ if (suffix.size() > str.size())
+ {
+ return false;
+ }
+ return str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+Result<std::string_view> IPCManagerCompiler::readInternal(char (&buffer)[4096]) const
+{
+ std::string *output = nullptr;
+ while (true)
+ {
+ uint64_t bytesRead = 0;
+#ifdef _WIN32
+ DWORD readCount = 0;
+ const bool success = ReadFile(GetStdHandle(STD_INPUT_HANDLE), buffer, sizeof(buffer), &readCount, nullptr);
+
+ bytesRead = readCount;
+ if (const DWORD lastError = GetLastError(); !success && lastError != ERROR_MORE_DATA)
+ {
+ return Error{getErrorString()};
+ }
+
+#else
+ bytesRead = read(STDIN_FILENO, buffer, 4096);
+ if (bytesRead == UINT64_MAX)
+ {
+ if (errno == EINTR)
+ {
+ continue;
+ }
+ return Error{getErrorString()};
+ }
+
+#endif
+ if (!bytesRead)
+ {
+ return Error{getErrorString(ErrorCategory::READ_FILE_ZERO_BYTES_READ)};
+ }
+
+ if (!output)
+ {
+ output = allocations.emplace_back(std::make_unique<std::string>()).get();
+ }
+
+ output->append(buffer, bytesRead);
+
+ // Retain the complete response: cached names and paths borrow its bytes.
+ if (endsWith(*output, delimiter))
+ {
+ return std::string_view{output->data(), output->size() - strlen(delimiter)};
+ }
+ }
+}
+
+Result<void> IPCManagerCompiler::writeInternal(const std::string_view buffer) const
+{
+#ifdef _WIN32
+ return writeAll(GetStdHandle(STD_OUTPUT_HANDLE), buffer);
+#else
+ return writeAll(STDOUT_FILENO, buffer.data(), buffer.size());
+#endif
+}
+
+Result<Response> IPCManagerCompiler::readBMIResponse(const std::string_view message, uint64_t &bytesRead,
+ const FileType type, const bool isSystem)
+{
+ TRY_READ_VAL(filePath, readPath, message, bytesRead);
+ TRY_READ_VAL(contents, loadBMIContents, filePath);
+ return Response{filePath, contents, type, isSystem};
+}
+
+Result<std::string_view> IPCManagerCompiler::loadBMIContents(const std::string_view filePath)
+{
+ if (const auto it = bmiContentsByPath.find(filePath); it != bmiContentsByPath.end())
+ {
+ return it->second;
+ }
+
+ TRY_READ_VAL(contents, mapBMIFile, filePath);
+ bmiContentsByPath.emplace(filePath, contents);
+ return contents;
+}
+
+Result<std::string_view> IPCManagerCompiler::findBMIContents(const std::string_view filePath) const
+{
+ const auto it = bmiContentsByPath.find(filePath);
+ if (it == bmiContentsByPath.end())
+ {
+ return Error{std::string("BMI was not supplied by the build system: ") + std::string(filePath)};
+ }
+ return it->second;
+}
+
+Result<void> IPCManagerCompiler::readLogicalNames(const std::string_view message, uint64_t &bytesRead,
+ const Response &response)
+{
+ TRY_READ_VAL(logicalNamesSize, readUInt32, message, bytesRead);
+ for (uint64_t i = 0; i < logicalNamesSize; ++i)
+ {
+ TRY_READ_VAL(logicalName, readString, message, bytesRead);
+ responses.emplace(logicalName, response);
+ }
+ return {};
+}
+
+Result<void> IPCManagerCompiler::receiveBTCModule(const CTBModule &moduleName)
+{
+ std::string buffer = getBufferWithType(CTB::MODULE);
+ writeString(buffer, moduleName.moduleName);
+ writeUInt32(buffer, buffer.size());
+ buffer.append(delimiter, strlen(delimiter));
+ if (const auto &r = writeInternal(buffer); !r)
+ {
+ return Error{r.error()};
+ }
+
+ char stackBuffer[4096];
+ auto received = readInternal(stackBuffer);
+
+ if (!received)
+ {
+ return Error{received.error()};
+ }
+ const std::string_view message = *received;
+
+ uint64_t bytesRead = 0;
+
+ TRY_READ_VAL(requested, readBMIResponse, message, bytesRead, FileType::MODULE);
+ TRY_READ_VAL(isSystem, readBool, message, bytesRead);
+
+ const auto &str = allocations.emplace_back(std::make_unique<std::string>(moduleName.moduleName));
+ responses.emplace(*str, Response{requested.filePath, requested.bmiContents, FileType::MODULE, isSystem});
+
+ TRY_READ_VAL(modDepsSize, readUInt32, message, bytesRead);
+
+ for (uint64_t i = 0; i < modDepsSize; ++i)
+ {
+ TRY_READ_VAL(isHeaderUnit, readBool, message, bytesRead);
+ const FileType type = isHeaderUnit ? FileType::HEADER_UNIT : FileType::MODULE;
+ TRY_READ_VAL(modDep, readBMIResponse, message, bytesRead, type);
+ TRY_READ_VAL(isSystemDep, readBool, message, bytesRead);
+ TRY_READ(aliases, readLogicalNames, message, bytesRead,
+ (Response{modDep.filePath, modDep.bmiContents, type, isSystemDep}));
+ }
+
+ if (message.size() != bytesRead)
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ return {};
+}
+
+Result<void> IPCManagerCompiler::receiveBTCNonModule(const CTBNonModule &nonModule)
+{
+ std::string buffer = getBufferWithType(CTB::NON_MODULE);
+ buffer.push_back(nonModule.isHeaderUnit);
+ writeString(buffer, nonModule.logicalName);
+ writeUInt32(buffer, buffer.size());
+ buffer.append(delimiter, strlen(delimiter));
+ if (const auto &r = writeInternal(buffer); !r)
+ {
+ return Error{r.error()};
+ }
+
+ char stackBuffer[4096];
+ auto received = readInternal(stackBuffer);
+
+ if (!received)
+ {
+ return Error{received.error()};
+ }
+
+ std::string_view readCompilerMessage = *received;
+ uint64_t bytesRead = 0;
+
+ TRY_READ_VAL(isHeaderUnit, readBool, readCompilerMessage, bytesRead);
+ TRY_READ_VAL(isSystem, readBool, readCompilerMessage, bytesRead);
+ TRY_READ_VAL(headerFilesSize, readUInt32, readCompilerMessage, bytesRead);
+
+ for (uint64_t i = 0; i < headerFilesSize; ++i)
+ {
+ TRY_READ_VAL(logicalName, readString, readCompilerMessage, bytesRead);
+ TRY_READ_VAL(filePath, readPath, readCompilerMessage, bytesRead);
+ TRY_READ_VAL(isSystemHeaderFile, readBool, readCompilerMessage, bytesRead);
+
+ responses.emplace(logicalName, Response{filePath, {}, FileType::HEADER_FILE, isSystemHeaderFile});
+ }
+
+ const auto &str = allocations.emplace_back(std::make_unique<std::string>(nonModule.logicalName));
+ if (!isHeaderUnit)
+ {
+ TRY_READ_VAL(filePath, readPath, readCompilerMessage, bytesRead);
+ responses.emplace(*str, Response{filePath, {}, FileType::HEADER_FILE, isSystem});
+ if (readCompilerMessage.size() != bytesRead)
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ return {};
+ }
+
+ TRY_READ_VAL(file, readBMIResponse, readCompilerMessage, bytesRead, FileType::HEADER_UNIT, isSystem);
+ responses.emplace(*str, file);
+
+ TRY_READ(logicalNames, readLogicalNames, readCompilerMessage, bytesRead, file);
+
+ TRY_READ_VAL(huDepsSize, readUInt32, readCompilerMessage, bytesRead);
+ for (uint64_t i = 0; i < huDepsSize; ++i)
+ {
+ TRY_READ_VAL(huDep, readBMIResponse, readCompilerMessage, bytesRead, FileType::HEADER_UNIT);
+ TRY_READ_VAL(huDepIsSystem, readBool, readCompilerMessage, bytesRead);
+ TRY_READ(aliases, readLogicalNames, readCompilerMessage, bytesRead,
+ (Response{huDep.filePath, huDep.bmiContents, FileType::HEADER_UNIT, huDepIsSystem}));
+ }
+
+ if (readCompilerMessage.size() != bytesRead)
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ return {};
+}
+
+Result<Response> IPCManagerCompiler::findResponse(const std::string_view logicalName, const FileType type)
+{
+#ifdef _WIN32
+ std::string logicalName2{logicalName};
+ if (type != FileType::MODULE)
+ {
+ for (char &c : logicalName2)
+ {
+ c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
+ }
+ }
+#else
+ const std::string_view logicalName2 = logicalName;
+#endif
+
+ // Include translation permits a header unit to satisfy a textual-header request.
+ if (const auto &it = responses.find(logicalName2);
+ it == responses.end() ||
+ (it->second.type != type && (it->second.type != FileType::HEADER_UNIT || type != FileType::HEADER_FILE)))
+ {
+ if (isMocking)
+ {
+ return Error{"Could not find entry in mocking-mode"};
+ }
+
+ if (type == FileType::MODULE)
+ {
+ CTBModule ctbModule;
+ ctbModule.moduleName = logicalName2;
+ if (const auto &r2 = receiveBTCModule(ctbModule); !r2)
+ {
+ return Error{r2.error()};
+ }
+ }
+ else
+ {
+ CTBNonModule ctbNonModule;
+ ctbNonModule.logicalName = logicalName2;
+ ctbNonModule.isHeaderUnit = type == FileType::HEADER_UNIT;
+ if (const auto &r2 = receiveBTCNonModule(ctbNonModule); !r2)
+ {
+ return Error{r2.error()};
+ }
+ }
+
+ return responses.at(logicalName2);
+ }
+ else
+ {
+ return it->second;
+ }
+}
+
+static Result<std::string> fileToString(const std::string_view fileName)
+{
+ std::ifstream file(std::string(fileName), std::ios::binary);
+ if (!file)
+ {
+ return Error{std::string("Could not open IPC mock file: ") + std::string(fileName)};
+ }
+ std::string fileBuffer{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()};
+ if (file.bad())
+ {
+ return Error{std::string("Could not read IPC mock file: ") + std::string(fileName)};
+ }
+ return fileBuffer;
+}
+
+Result<void> IPCManagerCompiler::readEntriesFromFile(const std::string_view filePath)
+{
+ // Neither a second mock nor a switch from live IPC may replace storage borrowed by cached entries.
+ if (isMocking || !responses.empty() || !allocations.empty() || !bmiContentsByPath.empty())
+ {
+ return Error{"IPC mock dependencies can only be loaded once on a fresh manager"};
+ }
+ isMocking = true;
+ auto contents = fileToString(filePath);
+ if (!contents)
+ {
+ return Error{contents.error()};
+ }
+ scanCacheFileData = std::move(*contents);
+
+ uint64_t bytesRead = 0;
+ TRY_READ_VAL(entriesSize, readUInt32, scanCacheFileData, bytesRead);
+ for (uint64_t i = 0; i < entriesSize; ++i)
+ {
+ TRY_READ_VAL(responseKey, readString, scanCacheFileData, bytesRead);
+
+ TRY_READ_VAL(valueFilePath, readPath, scanCacheFileData, bytesRead);
+ TRY_READ_VAL(fileType, readUInt8, scanCacheFileData, bytesRead);
+ if (fileType > static_cast<uint8_t>(FileType::HEADER_FILE))
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ TRY_READ_VAL(isSystem, readBool, scanCacheFileData, bytesRead);
+
+ const FileType type = static_cast<FileType>(fileType);
+ std::string_view contents;
+ if (type != FileType::HEADER_FILE)
+ {
+ TRY_READ_VAL(mapped, loadBMIContents, valueFilePath);
+ contents = mapped;
+ }
+ responses.emplace(responseKey, Response{valueFilePath, contents, type, isSystem});
+ }
+
+ if (bytesRead != scanCacheFileData.size())
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+
+ mockFilePath = filePath;
+ return {};
+}
+
+Result<std::string_view> IPCManagerCompiler::mapBMIFile(const std::string_view filePath)
+{
+ // Own a terminated path; callers may supply arbitrary string_views.
+ const std::string path(filePath);
+#ifdef _WIN32
+ const HANDLE handle = CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr,
+ OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+ if (handle == INVALID_HANDLE_VALUE)
+ {
+ return Error{getErrorString()};
+ }
+
+ LARGE_INTEGER size;
+ if (!GetFileSizeEx(handle, &size))
+ {
+ const std::string error = getErrorString();
+ CloseHandle(handle);
+ return Error{error};
+ }
+ if (size.QuadPart <= 0 || static_cast<uint64_t>(size.QuadPart) > (std::numeric_limits<size_t>::max)())
+ {
+ CloseHandle(handle);
+ return Error{std::string("Invalid BMI file size: ") + path};
+ }
+
+ // No name or build-system-owned object is needed. All readers open the completed file.
+ const HANDLE mapping = CreateFileMappingA(handle, nullptr, PAGE_READONLY, 0, 0, nullptr);
+ if (!mapping)
+ {
+ const std::string error = getErrorString();
+ CloseHandle(handle);
+ return Error{error};
+ }
+ CloseHandle(handle);
+
+ const void *view = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
+ if (!view)
+ {
+ const std::string error = getErrorString();
+ CloseHandle(mapping);
+ return Error{error};
+ }
+ // The view holds its own reference to the mapping object. Keep the view until process exit.
+ CloseHandle(mapping);
+ return std::string_view{static_cast<const char *>(view), static_cast<size_t>(size.QuadPart)};
+#else
+ const int fd = open(path.c_str(), O_RDONLY);
+ if (fd == -1)
+ {
+ return Error{getErrorString()};
+ }
+
+ struct stat st;
+ if (fstat(fd, &st) == -1)
+ {
+ const std::string error = getErrorString();
+ close(fd);
+ return Error{error};
+ }
+ if (st.st_size <= 0 || static_cast<uint64_t>(st.st_size) > (std::numeric_limits<size_t>::max)())
+ {
+ close(fd);
+ return Error{std::string("Invalid BMI file size: ") + path};
+ }
+ const size_t size = static_cast<size_t>(st.st_size);
+ int flags = MAP_SHARED;
+#ifdef MAP_POPULATE
+ flags |= MAP_POPULATE;
+#endif
+ void *view = mmap(nullptr, size, PROT_READ, flags, fd, 0);
+ if (view == MAP_FAILED)
+ {
+ const std::string error = getErrorString();
+ close(fd);
+ return Error{error};
+ }
+ close(fd);
+ // Closing the descriptor leaves the view valid; process exit releases the mapping.
+ return std::string_view{static_cast<const char *>(view), size};
+#endif
+}
+
+} // namespace P2978
diff --git a/clang/lib/IPC2978/Manager.cpp b/clang/lib/IPC2978/Manager.cpp
new file mode 100644
index 00000000000000..5ff8fee140732d
--- /dev/null
+++ b/clang/lib/IPC2978/Manager.cpp
@@ -0,0 +1,283 @@
+
+#include "clang/IPC2978/Manager.hpp"
+#include "clang/IPC2978/Messages.hpp"
+#include <algorithm>
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+
+#ifdef _WIN32
+#include <Windows.h>
+#else
+#include <limits.h>
+#include <unistd.h>
+#endif
+
+namespace P2978
+{
+
+std::string getErrorString()
+{
+#ifdef _WIN32
+ const DWORD err = GetLastError();
+
+ char *msg_buf = nullptr;
+ FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr,
+ err, MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), reinterpret_cast<char *>(&msg_buf), 0, nullptr);
+
+ if (msg_buf == nullptr)
+ {
+ char fallback_msg[128] = {};
+ snprintf(fallback_msg, sizeof(fallback_msg), "GetLastError() = %ld", err);
+ return fallback_msg;
+ }
+
+ std::string msg = msg_buf;
+ LocalFree(msg_buf);
+ return msg;
+#else
+ return {std::strerror(errno)};
+#endif
+}
+
+std::string getErrorString(const uint64_t bytesRead_, const uint64_t bytesProcessed_)
+{
+ return "Error: Message size does not match the parsed byte count.\nBytes Read: " + std::to_string(bytesRead_) +
+ ", Bytes Processed: " + std::to_string(bytesProcessed_);
+}
+
+std::string getErrorString(const ErrorCategory errorCategory_)
+{
+ std::string errorString;
+
+ switch (errorCategory_)
+ {
+ case ErrorCategory::PARSING_ERROR:
+ errorString = "P2978 Message Parsing Error.";
+ break;
+ case ErrorCategory::READ_FILE_ZERO_BYTES_READ:
+ errorString = "Compiler input closed before a complete IPC response was received.";
+ break;
+ case ErrorCategory::UNKNOWN_CTB_TYPE:
+ errorString = "Error: Unknown CTB message received.";
+ break;
+ case ErrorCategory::NONE:
+ std::string str = __FILE__;
+ str += ':';
+ str += std::to_string(__LINE__);
+ errorString = "P2978 IPC API internal error at " + str;
+ break;
+ }
+
+ return errorString;
+}
+
+#ifndef _WIN32
+Result<void> Manager::writeAll(const int fd, const char *buffer, const uint64_t count)
+{
+ uint64_t bytesWritten = 0;
+
+ while (bytesWritten != count)
+ {
+ // Converting the syscall's -1 result to uint64_t gives UINT64_MAX.
+ const uint64_t result = write(fd, buffer + bytesWritten, std::min<uint64_t>(count - bytesWritten, SSIZE_MAX));
+ if (result == UINT64_MAX)
+ {
+ if (errno == EINTR)
+ {
+ continue;
+ }
+ return Error{getErrorString()};
+ }
+ if (result == 0)
+ {
+ // Stop if the pipe makes no progress; errno need not describe this case.
+ return Error{"write returned zero bytes"};
+ }
+ bytesWritten += result;
+ }
+
+ return {};
+}
+#else
+Result<void> Manager::writeAll(void *handle, std::string_view buffer)
+{
+ uint64_t offset = 0;
+ while (offset < buffer.size())
+ {
+ const DWORD count = static_cast<DWORD>(std::min<uint64_t>(buffer.size() - offset, MAXDWORD));
+ DWORD written = 0;
+ if (!WriteFile(handle, buffer.data() + offset, count, &written, nullptr))
+ {
+ return Error{getErrorString()};
+ }
+ if (!written)
+ {
+ return Error{"WriteFile wrote zero bytes"};
+ }
+ offset += written;
+ }
+ return {};
+}
+#endif
+
+std::string Manager::getBufferWithType(CTB type)
+{
+ std::string buffer;
+ buffer.push_back(static_cast<uint8_t>(type));
+ return buffer;
+}
+
+void Manager::writeUInt32(std::string &buffer, const uint32_t value)
+{
+ const auto ptr = reinterpret_cast<const char *>(&value);
+ buffer.append(ptr, sizeof(value));
+}
+
+void Manager::writeString(std::string &buffer, const std::string_view &str)
+{
+ writeUInt32(buffer, str.size());
+ buffer.append(str.begin(), str.end());
+}
+
+void Manager::writePath(std::string &buffer, const std::string_view &str)
+{
+ writeString(buffer, str);
+ buffer.push_back('\0');
+}
+
+void Manager::writeModuleDep(std::string &buffer, const ModuleDep &dep)
+{
+ buffer.push_back(dep.isHeaderUnit);
+ writePath(buffer, dep.filePath);
+ buffer.push_back(dep.isSystem);
+ writeVectorOfStrings(buffer, dep.logicalNames);
+}
+
+void Manager::writeHuDep(std::string &buffer, const HuDep &dep)
+{
+ writePath(buffer, dep.filePath);
+ buffer.push_back(dep.isSystem);
+ writeVectorOfStrings(buffer, dep.logicalNames);
+}
+
+void Manager::writeHeaderFile(std::string &buffer, const HeaderFile &dep)
+{
+ writeString(buffer, dep.logicalName);
+ writePath(buffer, dep.filePath);
+ buffer.push_back(dep.isSystem);
+}
+
+void Manager::writeVectorOfStrings(std::string &buffer, const std::vector<std::string_view> &strs)
+{
+ writeUInt32(buffer, strs.size());
+ for (const std::string_view &str : strs)
+ {
+ writeString(buffer, str);
+ }
+}
+
+void Manager::writeVectorOfModuleDep(std::string &buffer, const std::vector<ModuleDep> &deps)
+{
+ writeUInt32(buffer, deps.size());
+ for (const ModuleDep &dep : deps)
+ {
+ writeModuleDep(buffer, dep);
+ }
+}
+
+void Manager::writeVectorOfHuDeps(std::string &buffer, const std::vector<HuDep> &deps)
+{
+ writeUInt32(buffer, deps.size());
+ for (const HuDep &dep : deps)
+ {
+ writeHuDep(buffer, dep);
+ }
+}
+
+void Manager::writeVectorOfHeaderFiles(std::string &buffer, const std::vector<HeaderFile> &headerFiles)
+{
+ writeUInt32(buffer, headerFiles.size());
+ for (const HeaderFile &headerFile : headerFiles)
+ {
+ writeHeaderFile(buffer, headerFile);
+ }
+}
+
+Result<bool> Manager::readBool(const std::string_view message, uint64_t &bytesRead)
+{
+ if (bytesRead >= message.size())
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ const unsigned char value = static_cast<unsigned char>(message[bytesRead]);
+ if (value > 1)
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ bool result = value != 0;
+ bytesRead += 1;
+ return result;
+}
+
+Result<uint8_t> Manager::readUInt8(const std::string_view message, uint64_t &bytesRead)
+{
+ if (bytesRead >= message.size())
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ uint8_t result = *reinterpret_cast<const uint8_t *>(message.data() + bytesRead);
+ bytesRead += 1;
+ return result;
+}
+
+Result<uint32_t> Manager::readUInt32(const std::string_view message, uint64_t &bytesRead)
+{
+ if (bytesRead > message.size() || message.size() - bytesRead < sizeof(uint32_t))
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ uint32_t result;
+ memcpy(&result, message.data() + bytesRead, sizeof(result));
+ bytesRead += sizeof(result);
+ return result;
+}
+
+Result<std::string_view> Manager::readString(const std::string_view message, uint64_t &bytesRead)
+{
+ auto r = readUInt32(message, bytesRead);
+ if (!r)
+ {
+ return Error{r.error()};
+ }
+ const uint64_t stringSize = *r;
+ if (bytesRead > message.size() || stringSize > message.size() - bytesRead)
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ std::string_view result = {message.data() + bytesRead, stringSize};
+ bytesRead += stringSize;
+ return result;
+}
+
+Result<std::string_view> Manager::readPath(const std::string_view message, uint64_t &bytesRead)
+{
+ auto r = readUInt32(message, bytesRead);
+ if (!r)
+ {
+ return Error{r.error()};
+ }
+ const uint64_t stringSize = *r;
+ if (bytesRead > message.size() || stringSize >= message.size() - bytesRead ||
+ message[bytesRead + stringSize] != '\0')
+ {
+ return Error{getErrorString(ErrorCategory::PARSING_ERROR)};
+ }
+ std::string_view result = {message.data() + bytesRead, stringSize};
+ bytesRead += stringSize;
+ // Consume the terminator without including it in the returned view.
+ bytesRead += 1;
+ return result;
+}
+
+} // namespace P2978
diff --git a/clang/lib/IPC2978/setup.py b/clang/lib/IPC2978/setup.py
new file mode 100755
index 00000000000000..d21b1a28aefac9
--- /dev/null
+++ b/clang/lib/IPC2978/setup.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Refresh Clang's IPC2978 copy from the sibling ipc2978api repository."""
+import argparse
+from pathlib import Path
+import subprocess
+import sys
+
+
+def main():
+ repo = Path(__file__).resolve().parents[3]
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--source', type=Path, default=repo.parent / 'ipc2978api')
+ args = parser.parse_args()
+ subprocess.run([
+ sys.executable, str(args.source / 'tools/copy_library.py'),
+ '--include-dir', str(repo / 'clang/include/clang/IPC2978'),
+ '--source-dir', str(repo / 'clang/lib/IPC2978'),
+ '--include-prefix', 'clang/IPC2978/',
+ '--clang-tests-dir', str(repo / 'clang/unittests/IPC2978'),
+ ], check=True)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/clang/lib/Lex/CMakeLists.txt b/clang/lib/Lex/CMakeLists.txt
index 7b0be7249cd99a..e4446faa35d457 100644
--- a/clang/lib/Lex/CMakeLists.txt
+++ b/clang/lib/Lex/CMakeLists.txt
@@ -35,4 +35,5 @@ add_clang_library(clangLex
LINK_LIBS
clangBasic
+ clangIPC2978
)
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index f1e9aaa72ff048..6752a2bba0639c 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -23,6 +23,7 @@
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Basic/TokenKinds.h"
+#include "clang/IPC2978/IPCManagerCompiler.hpp"
#include "clang/Lex/CodeCompletionHandler.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/LexDiagnostic.h"
@@ -2404,7 +2405,6 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
SmallString<1024> RelativePath;
// We get the raw path only if we have 'Callbacks' to which we later pass
// the path.
- ModuleMap::KnownHeader SuggestedModule;
SourceLocation FilenameLoc = FilenameTok.getLocation();
StringRef LookupFilename = Filename;
@@ -2419,10 +2419,44 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
BackslashStyle = llvm::sys::path::Style::windows;
}
- OptionalFileEntryRef File = LookupHeaderIncludeOrImport(
- &CurDir, Filename, FilenameLoc, FilenameRange, FilenameTok,
- IsFrameworkFound, IsImportDecl, IsMapped, LookupFrom, LookupFromFile,
- LookupFilename, RelativePath, SearchPath, SuggestedModule, isAngled);
+ ModuleMap::KnownHeader SuggestedModule;
+ OptionalFileEntryRef File;
+
+ if (P2978::managerCompiler) {
+ const auto &Result = P2978::managerCompiler->findResponse(
+ Filename, IsImportDecl ? P2978::FileType::HEADER_UNIT
+ : P2978::FileType::HEADER_FILE);
+ if (!Result) {
+ Diag(FilenameLoc, getDiagnostics().getCustomDiagID(
+ DiagnosticsEngine::Error,
+ "could not resolve IPC dependency '%0': %1"))
+ << Filename << Result.error();
+ return {ImportAction::None};
+ }
+ File = getFileManager().getOptionalFileRef(Result->filePath);
+ if (!File) {
+ Diag(FilenameTok, diag::err_pp_file_not_found)
+ << StringRef(Result->filePath);
+ return {ImportAction::None};
+ }
+ if (Result->isSystem) {
+ getHeaderSearchInfo().MarkFileSystemHeader(*File);
+ }
+ if (Result->type == P2978::FileType::HEADER_UNIT) {
+ Module *HeaderUnit =
+ getModuleLoader().loadIPCReceivedHeaderUnit(Result->filePath, EndLoc);
+ if (!HeaderUnit) {
+ return {ImportAction::None};
+ }
+ IsImportDecl = true;
+ SuggestedModule = {HeaderUnit, ModuleMap::NormalHeader};
+ }
+ } else {
+ File = LookupHeaderIncludeOrImport(
+ &CurDir, Filename, FilenameLoc, FilenameRange, FilenameTok,
+ IsFrameworkFound, IsImportDecl, IsMapped, LookupFrom, LookupFromFile,
+ LookupFilename, RelativePath, SearchPath, SuggestedModule, isAngled);
+ }
if (usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) {
if (File && isPCHThroughHeader(&File->getFileEntry()))
@@ -2484,8 +2518,12 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
// Determine whether we should try to import the module for this #include, if
// there is one. Don't do so if precompiled module support is disabled or we
- // are processing this module textually (because we're building the module).
- if (MaybeTranslateInclude && (UsableHeaderUnit || UsableClangHeaderModule)) {
+ // are processing this module textually (because we're building the module)
+ // or when useIPC is set as header-unit is already loaded.
+ if (P2978::managerCompiler && IsImportDecl) {
+ Action = Import;
+ } else if (MaybeTranslateInclude &&
+ (UsableHeaderUnit || UsableClangHeaderModule)) {
// If this include corresponds to a module but that module is
// unavailable, diagnose the situation and bail out.
// FIXME: Remove this; loadModule does the same check (but produces
@@ -2634,9 +2672,11 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
}
// Issue a diagnostic if the name of the file on disk has a different case
- // than the one we're about to open.
+ // than the one we're about to open. Not checked if it is an IPC received
+ // module.
const bool CheckIncludePathPortability =
- !IsMapped && !File->getFileEntry().tryGetRealPathName().empty();
+ !(P2978::managerCompiler && IsImportDecl) && !IsMapped &&
+ !File->getFileEntry().tryGetRealPathName().empty();
if (CheckIncludePathPortability) {
StringRef Name = LookupFilename;
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index c86ed6b2ea3f98..10742503fbe712 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -662,11 +662,17 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result,
Module *Mod = reinterpret_cast<Module *>(Tok.getAnnotationValue());
// FIXME: We need a better way to disambiguate C++ clang modules and
// standard C++ modules.
- if (!getLangOpts().CPlusPlusModules || !Mod->isHeaderUnit())
+ if (!getLangOpts().CPlusPlusModules || !Mod->isHeaderUnit()) {
Actions.ActOnAnnotModuleInclude(Loc, Mod);
- else {
+ } else {
+ // Preserve transitive #include visibility when building a header unit.
+ // Explicit import declarations keep their own export semantics.
+ SourceLocation ExportLoc =
+ getLangOpts().getCompilingModule() == LangOptions::CMK_HeaderUnit
+ ? Loc
+ : SourceLocation();
DeclResult Import =
- Actions.ActOnModuleImport(Loc, SourceLocation(), Loc, Mod);
+ Actions.ActOnModuleImport(Loc, ExportLoc, Loc, Mod);
Decl *ImportDecl = Import.isInvalid() ? nullptr : Import.get();
Result = Actions.ConvertDeclToDeclGroup(ImportDecl);
}
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 86d419ce6e7abe..f2272e7d5a90e1 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -68,6 +68,7 @@
#include "clang/Basic/TargetOptions.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Basic/Version.h"
+#include "clang/IPC2978/IPCManagerCompiler.hpp"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/HeaderSearchOptions.h"
#include "clang/Lex/MacroInfo.h"
@@ -3530,9 +3531,27 @@ ASTReader::ReadControlBlock(ModuleFile &F,
// explicit name to file mappings. Also, we will still verify the
// size/signature making sure it is essentially the same file but
// perhaps in a different location.
- if (ImportedKind == MK_PrebuiltModule || ImportedKind == MK_ExplicitModule)
- ImportedFile = PP.getHeaderSearchInfo().getPrebuiltModuleFileName(
- ImportedName, /*FileMapOnly*/ !IsImportingStdCXXModule);
+ if (ImportedKind == MK_PrebuiltModule ||
+ ImportedKind == MK_ExplicitModule) {
+ if (P2978::managerCompiler) {
+ if (IsImportingStdCXXModule) {
+ const auto &Result = P2978::managerCompiler->findResponse(
+ ImportedName, P2978::FileType::MODULE);
+ if (!Result) {
+ Diag(CurrentImportLoc,
+ getDiags().getCustomDiagID(
+ DiagnosticsEngine::Error,
+ "could not resolve IPC dependency '%0': %1"))
+ << ImportedName << Result.error();
+ return Failure;
+ }
+ ImportedFile = ModuleFileName::makeExplicit(Result->filePath);
+ }
+ } else {
+ ImportedFile = PP.getHeaderSearchInfo().getPrebuiltModuleFileName(
+ ImportedName, /*FileMapOnly*/ !IsImportingStdCXXModule);
+ }
+ }
if (IsImportingStdCXXModule && ImportedFile.empty()) {
Diag(diag::err_failed_to_find_module_file) << ImportedName;
diff --git a/clang/lib/Serialization/ModuleManager.cpp b/clang/lib/Serialization/ModuleManager.cpp
index cbb9de32f2b15b..86f27cd9a88641 100644
--- a/clang/lib/Serialization/ModuleManager.cpp
+++ b/clang/lib/Serialization/ModuleManager.cpp
@@ -14,6 +14,7 @@
#include "clang/Serialization/ModuleManager.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/LLVM.h"
+#include "clang/IPC2978/IPCManagerCompiler.hpp"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/ModuleMap.h"
#include "clang/Serialization/GlobalModuleIndex.h"
@@ -55,9 +56,10 @@ ModuleManager::makeKey(const ModuleFileName &Name) const {
ModCache.getInMemoryModuleCache().lookupPCM(Name, Size, ModTime))
return ModuleFileKey(Buf);
} else {
- if (auto ModuleFile = FileMgr.getOptionalFileRef(Name, /*OpenFile=*/true,
- /*CacheFailure=*/true,
- /*IsText=*/false))
+ // IPC has already mapped the BMI; retain its identity without opening it again.
+ if (auto ModuleFile = FileMgr.getOptionalFileRef(
+ Name, /*OpenFile=*/!P2978::managerCompiler, /*CacheFailure=*/true,
+ /*IsText=*/false))
return ModuleFileKey(*ModuleFile);
}
@@ -205,7 +207,8 @@ AddModuleResult ModuleManager::addModule(
Expected<FileEntryRef> Entry =
FileName == StringRef("-")
? FileMgr.getSTDIN()
- : FileMgr.getFileRef(FileName, /*OpenFile=*/true,
+ : FileMgr.getFileRef(FileName,
+ /*OpenFile=*/!P2978::managerCompiler,
/*CacheFailure=*/true,
/*IsText=*/false);
if (!Entry)
@@ -214,6 +217,20 @@ AddModuleResult ModuleManager::addModule(
Size = Entry->getSize();
ModTime = Entry->getModificationTime();
+ if (P2978::managerCompiler) {
+ const StringRef EntryName = Entry->getName();
+ const auto Contents =
+ P2978::managerCompiler->findBMIContents(EntryName);
+ if (!Contents) {
+ return llvm::createStringError(llvm::inconvertibleErrorCode(),
+ Contents.error());
+ }
+ // IPC BMI views remain valid until process exit and need no trailing
+ // NUL.
+ return llvm::MemoryBuffer::getMemBuffer(
+ *Contents, EntryName, /*RequiresNullTerminator=*/false);
+ }
+
// RequiresNullTerminator is false because module files don't need it, and
// this allows the file to still be mmapped.
return llvm::errorOrToExpected(
diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt
index 5cc21a563be3e5..c970f9729c7cd9 100644
--- a/clang/unittests/CMakeLists.txt
+++ b/clang/unittests/CMakeLists.txt
@@ -94,6 +94,9 @@ add_subdirectory(Rewrite)
add_subdirectory(ScalableStaticAnalysis)
add_subdirectory(Sema)
add_subdirectory(CodeGen)
+if(TARGET clang)
+ add_subdirectory(IPC2978)
+endif()
if(HAVE_CLANG_REPL_SUPPORT)
add_subdirectory(Interpreter)
endif()
diff --git a/clang/unittests/IPC2978/.clang-format-ignore b/clang/unittests/IPC2978/.clang-format-ignore
new file mode 100644
index 00000000000000..f59ec20aabf584
--- /dev/null
+++ b/clang/unittests/IPC2978/.clang-format-ignore
@@ -0,0 +1 @@
+*
\ No newline at end of file
diff --git a/clang/unittests/IPC2978/CMakeLists.txt b/clang/unittests/IPC2978/CMakeLists.txt
new file mode 100644
index 00000000000000..9cc2c689e2c36c
--- /dev/null
+++ b/clang/unittests/IPC2978/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_distinct_clang_unittest(IPC2978Tests
+ IPC2978Test.cpp
+ CLANG_LIBS
+ clangIPC2978
+ LLVM_COMPONENTS
+ Support
+ )
+
+target_compile_definitions(IPC2978Tests PRIVATE
+ LLVM_TOOLS_BINARY_DIR="${LLVM_TOOLS_BINARY_DIR}"
+ )
+add_dependencies(IPC2978Tests clang)
diff --git a/clang/unittests/IPC2978/IPC2978Test.cpp b/clang/unittests/IPC2978/IPC2978Test.cpp
new file mode 100644
index 00000000000000..2361379c259cb8
--- /dev/null
+++ b/clang/unittests/IPC2978/IPC2978Test.cpp
@@ -0,0 +1,819 @@
+//===- unittests/IPC2978/IPC2978Test.cpp - Tests IPC2978 Support -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// The following line is uncommented by clang/lib/IPC2978/setup.py for clang/unittests/IPC2978/IPC2978.cpp
+
+#define IS_THIS_CLANG_REPO
+
+#include <cstddef>
+#include <cstring>
+#include <iostream>
+#ifdef IS_THIS_CLANG_REPO
+#include "clang/IPC2978/IPCManagerCompiler.hpp"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+#include "gtest/gtest.h"
+#else
+#include "Testing.hpp"
+#endif
+
+#include "TestBuildSystem.hpp"
+#include "TestProcess.hpp"
+#include <filesystem>
+#include <fstream>
+
+using namespace std::filesystem;
+using namespace P2978;
+using namespace std;
+
+#ifdef _WIN32
+constexpr const char *clangExecutableName = "clang.exe";
+#else
+constexpr const char *clangExecutableName = "clang";
+#endif
+
+#ifndef IPC2978_CLANG_EXECUTABLE
+#define IPC2978_CLANG_EXECUTABLE ""
+#endif
+
+namespace
+{
+
+P2978::Result<string> compilerCommand(const path &compiler)
+{
+ std::error_code error;
+ const path executable = absolute(compiler, error);
+ if (error)
+ {
+ return P2978::Error{"Cannot resolve Clang executable: " + error.message()};
+ }
+ if (!is_regular_file(executable, error))
+ {
+ return P2978::Error{"Clang executable not found: " + executable.string() +
+ "\nPass the path to a Clang rebuilt with this IPC2978 library, or configure "
+ "IPC2978_CLANG_EXECUTABLE."};
+ }
+#ifndef _WIN32
+ if (access(executable.c_str(), X_OK) != 0)
+ {
+ return P2978::Error{"Clang file is not executable: " + executable.string()};
+ }
+#endif
+ string command = "\"";
+ for (const char c : executable.string())
+ {
+#ifndef _WIN32
+ // TestProcess uses wordexp on Unix, so preserve shell metacharacters literally.
+ if (c == '\\' || c == '"' || c == '$' || c == '`')
+ {
+ command += '\\';
+ }
+#endif
+ command += c;
+ }
+ command += '"';
+ return command;
+}
+
+struct CompilerSession
+{
+ ipc2978_test::TestProcess process;
+ std::string output;
+ CTB type{};
+ alignas(std::max_align_t) char buffer[320];
+
+ P2978::Result<void> readRequest()
+ {
+ if (!process.readCompilerMessage(output))
+ {
+ return P2978::Error{process.error.empty() ? "Compiler exited before the expected request:\n" + output
+ : process.error};
+ }
+ if (!process.pruneCompilerOutput(output, buffer, type))
+ {
+ return P2978::Error{process.error};
+ }
+ return {};
+ }
+
+ P2978::Result<ipc2978_test::TestBuildSystem> start(const std::string &command, bool requestExpected)
+ {
+ output.clear();
+ if (!process.startAsyncProcess(command.c_str()))
+ {
+ return P2978::Error{process.error};
+ }
+ if (requestExpected)
+ {
+ if (const auto result = readRequest(); !result)
+ {
+ return P2978::Error{result.error()};
+ }
+ }
+ return ipc2978_test::TestBuildSystem{process.writePipe};
+ }
+
+ P2978::Result<void> finish(int expectedExitStatus = EXIT_SUCCESS, string_view diagnostic = {})
+ {
+ if (process.readCompilerMessage(output))
+ {
+ return P2978::Error{std::string("Unexpected compiler request at completion")};
+ }
+ if (!process.error.empty())
+ {
+ return P2978::Error{process.error};
+ }
+ if (!process.reapProcess())
+ {
+ return P2978::Error{process.error};
+ }
+ if (process.exitStatus != expectedExitStatus)
+ {
+ return P2978::Error{"Compiler exited with status " + std::to_string(process.exitStatus) + " (expected " +
+ std::to_string(expectedExitStatus) + "):\n" + output};
+ }
+ if (!diagnostic.empty() && output.find(diagnostic) == string::npos)
+ {
+ return P2978::Error{"Missing compiler diagnostic: " + string(diagnostic) + "\n" + output};
+ }
+ std::cout << output;
+ return {};
+ }
+};
+
+// main.cpp
+const string mainDotCpp = R"(
+// only one request of Foo will be made as A and big.hpp
+// will be provided with it.
+import Foo;
+import A;
+#include "y.hpp"
+#include "z.hpp"
+
+int main()
+{
+ Hello();
+ World();
+ Foo();
+}
+)";
+
+// Creates all the input files (source files + pcm files) that are needed for the test.
+void setupTest()
+{
+ // a.cpp
+ const string aDotCpp = R"(
+export module A; // primary module interface unit
+
+export import :B; // Hello() is visible when importing 'A'.
+import :C; // WorldImpl() is now visible only for 'a.cpp'.
+// export import :C; // ERROR: Cannot export a module implementation unit.
+
+// World() is visible by any translation unit importing 'A'.
+export char const* World()
+{
+ return WorldImpl();
+}
+)";
+ // a-b.cpp
+ const string aBDotCPP = R"(
+export module A:B; // partition module interface unit
+
+// Hello() is visible by any translation unit importing 'A'.
+export char const* Hello() { return "Hello"; }
+)";
+
+ // a-c.cpp
+ const string aCDotCPP = R"(
+module A:C; // partition module implementation unit
+
+// WorldImpl() is visible by any module unit of 'A' importing ':C'.
+char const* WorldImpl() { return "World"; }
+)";
+
+ // m.hpp, n.hpp and o.hpp are to be used as header-units, header-files
+ // while x.hpp, y.hpp and z.hpp are to be used as big-hu by include big.hpp
+
+ // m.hpp
+ const string mDotHpp = R"(
+// this file can not be included without first defining M_HEADER_FILE
+// this is to demonstrate difference between header-file and header-unit.
+// as macros don't seep into header-units
+
+#ifdef M_HEADER_FILE
+inline int m = 5;
+#else
+fail compilation
+#endif
+)";
+
+ // n.hpp
+ const string nDotHpp = R"(
+// should work just fine as macro should not seep in here while inclusion.
+
+#ifdef N_HEADER_FILE
+fail compilation
+#else
+#define M_HEADER_FILE
+#include "m.hpp"
+inline int n = 5 + m;
+#endif
+
+// COMMAND_MACRO should be defined while compiling this.
+// however, it should still be fine if it is not defined while compiling
+// a file consuming this
+
+#ifndef COMMAND_MACRO
+fail compilation
+#endif
+)";
+
+ // o.hpp
+ const string oDotHpp = R"(
+// TRANSLATING should be defined if /translateInclude is being used.
+// "o.hpp" should still be treated as header-file.
+
+#define M_HEADER_FILE
+#include "m.hpp"
+#ifdef TRANSLATING
+#include "n.hpp"
+#else
+import "n.hpp";
+#endif
+
+inline int o = n + m + 5;
+)";
+
+ // x.hpp
+ const string xDotHpp = R"(
+#ifndef X_HPP
+#define X_HPP
+inline int x = 5;
+#endif
+)";
+
+ // y.hpp
+ const string yDotHpp = R"(
+#ifndef Y_HPP
+#define Y_HPP
+#include "x.hpp"
+inline int y = x + 5;
+#endif
+)";
+
+ // z.hpp
+ const string zDotHpp = R"(
+#ifndef Z_HPP
+#define Z_HPP
+#include "y.hpp"
+inline int z = x + y + 5;
+#endif
+)";
+
+ // big.hpp
+ const string bigDotHpp = R"(
+#include "x.hpp"
+// todo
+// following two should not be requested as big.hpp includes the following as well.
+#include "y.hpp"
+#include "z.hpp"
+)";
+
+ // foo.cpp
+ const string fooDotCpp = R"(
+module;
+#include "x.hpp"
+#include "z.hpp"
+#include "y.hpp"
+#include "big.hpp"
+export module Foo;
+import A;
+
+export void Foo()
+{
+ Hello();
+ World();
+ int s = x + y + z;
+}
+)";
+
+ ofstream("a.cpp") << aDotCpp;
+ ofstream("a-b.cpp") << aBDotCPP;
+ ofstream("a-c.cpp") << aCDotCPP;
+ ofstream("m.hpp") << mDotHpp;
+ ofstream("n.hpp") << nDotHpp;
+ ofstream("o.hpp") << oDotHpp;
+ ofstream("x.hpp") << xDotHpp;
+ ofstream("y.hpp") << yDotHpp;
+ ofstream("z.hpp") << zDotHpp;
+ ofstream("big.hpp") << bigDotHpp;
+ ofstream("foo.cpp") << fooDotCpp;
+ ofstream("main.cpp") << mainDotCpp;
+}
+
+P2978::Error errorReturn()
+{
+ return P2978::Error{"IPC2978 Test Error: Wrong Message Received\n"};
+}
+
+#define CHECK(condition) \
+ if (!(condition)) \
+ { \
+ return errorReturn(); \
+ }
+
+#define SEND_MESSAGE(message) \
+ if (const auto &_r_send_##message = manager.sendMessage(message); !_r_send_##message) \
+ { \
+ return P2978::Error{"manager send message failed" + _r_send_##message.error() + "\n"}; \
+ }
+
+#define CHECK_RESULT(expression) \
+ if (const auto &result = (expression); !result) \
+ { \
+ return P2978::Error{result.error()}; \
+ }
+
+P2978::Result<void> checkFailureCases(const string &clangCommand, string_view fooPcm, string_view bigPcm)
+{
+ struct MockEntry
+ {
+ string_view logicalName;
+ string_view filePath;
+ FileType type;
+ };
+ struct FailureCase
+ {
+ string_view source;
+ vector<MockEntry> entries;
+ string_view diagnostic;
+ };
+ const FailureCase cases[] = {
+ {"#include \"missing.hpp\"\n", {}, "could not resolve IPC dependency 'missing.hpp':"},
+ {"import Missing;\n", {}, "could not resolve IPC dependency 'Missing':"},
+ {"#include \"missing.hpp\"\n",
+ {{"missing.hpp", "ipc missing.hpp", FileType::HEADER_FILE}},
+ "'ipc missing.hpp' file not found"},
+ {"import \"corrupt.hpp\";\n",
+ {{"corrupt.hpp", "ipc corrupt.pcm", FileType::HEADER_UNIT}},
+ "file 'ipc corrupt.pcm' is not a valid"},
+ // Foo also imports a header unit; provide that mapping so loading reaches the missing named dependency A.
+ {"import Foo;\n",
+ {{"Foo", fooPcm, FileType::MODULE}, {"big.hpp", bigPcm, FileType::HEADER_UNIT}},
+ "could not resolve IPC dependency 'A':"},
+ };
+
+ remove(path("ipc missing.hpp"));
+ ofstream corrupt("ipc corrupt.pcm", ios::binary);
+ corrupt << "invalid BMI contents";
+ corrupt.close();
+ if (!corrupt)
+ {
+ return P2978::Error{"Could not write corrupt BMI fixture"};
+ }
+ CompilerSession session;
+ for (const auto &test : cases)
+ {
+ string mock;
+ Manager::writeUInt32(mock, static_cast<uint32_t>(test.entries.size()));
+ for (const auto &entry : test.entries)
+ {
+ Manager::writeString(mock, entry.logicalName);
+ Manager::writePath(mock, entry.filePath);
+ mock.push_back(static_cast<uint8_t>(entry.type));
+ mock.push_back(true);
+ }
+ ofstream source("ipc failure.cpp", ios::binary);
+ source << test.source;
+ source.close();
+ ofstream cache("ipc failure.cache", ios::binary);
+ cache << mock;
+ cache.close();
+ if (!source || !cache)
+ {
+ return P2978::Error{"Could not write IPC failure fixtures"};
+ }
+ CHECK_RESULT(session.start(
+ clangCommand + " -std=c++20 -fsyntax-only -useIPC=\"ipc failure.cache\" \"ipc failure.cpp\"", false))
+ // A crash or an unexpected live request must fail the test even if an error diagnostic was printed first.
+ CHECK_RESULT(session.finish(1, test.diagnostic))
+ }
+ return {};
+}
+
+P2978::Result<void> checkHeaderUnitVisibility(const string &clangCommand, const string &aPcm,
+ const string &aBPcm, const string &aCPcm)
+{
+ CompilerSession session;
+ auto compile = [&](const string &arguments, int exitStatus = 0, string_view diagnostic = {}) -> P2978::Result<void> {
+ CHECK_RESULT(session.start(clangCommand + " -std=c++20 " + arguments, false))
+ return session.finish(exitStatus, diagnostic);
+ };
+ const string headerPcm = absolute("ipc visibility.pcm").string();
+ const string dependencies = " -fmodule-file=A=\"" + aPcm + "\" -fmodule-file=A:B=\"" + aBPcm +
+ "\" -fmodule-file=A:C=\"" + aCPcm + "\"";
+
+ // Declarations introducing names in the header are implicitly exported.
+ // Importing named module A does not expose A's names to the header's consumers.
+ ofstream header("ipc visibility.hpp", ios::binary);
+ header << "import A;\ninline const char *headerWorld() { return World(); }\n";
+ header.close();
+ CHECK(header)
+ CHECK_RESULT(compile("-fmodule-header=user -xc++-header \"ipc visibility.hpp\" -o \"" + headerPcm +
+ "\"" + dependencies))
+
+ string mock;
+ Manager::writeUInt32(mock, 4);
+ const pair<string_view, string_view> entries[] = {
+ {"ipc visibility.hpp", headerPcm}, {"A", aPcm}, {"A:B", aBPcm}, {"A:C", aCPcm}};
+ for (const auto &[name, filePath] : entries)
+ {
+ Manager::writeString(mock, name);
+ Manager::writePath(mock, filePath);
+ mock.push_back(static_cast<uint8_t>(name == "ipc visibility.hpp" ? FileType::HEADER_UNIT : FileType::MODULE));
+ mock.push_back(true);
+ }
+ ofstream cache("ipc visibility.cache", ios::binary);
+ cache << mock;
+ cache.close();
+ CHECK(cache)
+
+ const pair<string_view, bool> cases[] = {
+ {"const char *result = headerWorld();\n", true},
+ {"const char *result = World();\n", false},
+ {"import A;\nconst char *result = World();\n", true}};
+ for (const auto &[declarations, succeeds] : cases)
+ {
+ ofstream source("ipc visibility.cpp", ios::binary);
+ source << "import \"ipc visibility.hpp\";\n" << declarations;
+ source.close();
+ CHECK(source)
+ CHECK_RESULT(compile("-fsyntax-only -useIPC=\"ipc visibility.cache\" \"ipc visibility.cpp\"",
+ succeeds ? 0 : 1,
+ succeeds ? "" : "declaration of 'World' must be imported from module 'A'"))
+ }
+ return {};
+}
+
+P2978::Result<void> runTest(const path &compiler)
+{
+ const auto command = compilerCommand(compiler);
+ CHECK_RESULT(command)
+ const string &clangCommand = *command;
+ CompilerSession session;
+ auto &type = session.type;
+ auto &buffer = session.buffer;
+ remove(path("main .o"));
+ setupTest();
+
+ string str = current_path().string();
+#ifdef _WIN32
+ for (char &c : str)
+ {
+ c = tolower(c);
+ }
+#endif
+
+ path curPath(str);
+
+ string mainFilePath = (curPath / "main .o").string();
+ string modFilePath = (curPath / "mod .pcm").string();
+ string mod1FilePath = (curPath / "mod1 .pcm").string();
+ string mod2FilePath = (curPath / "mod2 .pcm").string();
+ string aCObj = (curPath / "a-c .o").string();
+ string aCPcm = (curPath / "a-c .pcm").string();
+ string aBObj = (curPath / "a-b .o").string();
+ string aBPcm = (curPath / "a-b .pcm").string();
+ string aObj = (curPath / "a .o").string();
+ string aPcm = (curPath / "a .pcm").string();
+ string bObj = (curPath / "b .o").string();
+ string bPcm = (curPath / "b .pcm").string();
+ string mHpp = (curPath / "m.hpp").string();
+ string nHpp = (curPath / "n.hpp").string();
+ string oHpp = (curPath / "o.hpp").string();
+ string nPcm = (curPath / "n .pcm").string();
+ string oPcm = (curPath / "o .pcm").string();
+ string xHpp = (curPath / "x.hpp").string();
+ string yHpp = (curPath / "y.hpp").string();
+ string zHpp = (curPath / "z.hpp").string();
+ string bigHpp = (curPath / "big.hpp").string();
+ string bigPcm = (curPath / "big .pcm").string();
+ string fooPcm = (curPath / "foo .pcm").string();
+ string fooObj = (curPath / "foo .o").string();
+ string mainObj = (curPath / "main .o").string();
+
+ // compiling a-c.cpp
+ {
+ string compileCommand = clangCommand + R"( -std=c++20 -fmodules-reduced-bmi -o ")" + aCObj +
+ "\" -useIPC -c -xc++-module a-c.cpp -fmodule-output=\"" + aCPcm + "\"";
+
+ CHECK_RESULT(session.start(compileCommand, false))
+ CHECK_RESULT(session.finish())
+ }
+
+ // compiling a-b.cpp
+ {
+ string compileCommand = clangCommand + R"( -std=c++20 -fmodules-reduced-bmi -o ")" + aBObj +
+ "\" -useIPC -c -xc++-module a-b.cpp -fmodule-output=\"" + aBPcm + "\"";
+
+ CHECK_RESULT(session.start(compileCommand, false))
+ CHECK_RESULT(session.finish())
+ }
+ // compiling a.cpp
+ {
+ string compileCommand = clangCommand + R"( -std=c++20 -fmodules-reduced-bmi -o ")" + aObj +
+ "\" -useIPC -c -xc++-module a.cpp -fmodule-output=\"" + aPcm + "\"";
+
+ auto managerResult = session.start(compileCommand, true);
+ CHECK_RESULT(managerResult)
+ auto &manager = *managerResult;
+
+ CHECK(type == CTB::MODULE)
+ const auto &ctbModule = reinterpret_cast<CTBModule &>(buffer);
+ CHECK(ctbModule.moduleName == "A:B")
+
+ BTCModule btcMod;
+ btcMod.filePath = aBPcm;
+
+ ModuleDep modDep;
+ modDep.filePath = aCPcm;
+ modDep.logicalNames.emplace_back("A:C");
+ modDep.isHeaderUnit = false;
+ btcMod.modDeps.emplace_back(std::move(modDep));
+
+ SEND_MESSAGE(btcMod)
+
+ CHECK_RESULT(session.finish())
+ }
+
+ // compiling n.hpp
+ {
+ string compileCommand = clangCommand + R"( -std=c++20 -fmodule-header=user -o ")" + nPcm +
+ "\" -useIPC -xc++-header n.hpp -DCOMMAND_MACRO";
+
+ auto managerResult = session.start(compileCommand, true);
+ CHECK_RESULT(managerResult)
+ auto &manager = *managerResult;
+
+ CHECK(type == CTB::NON_MODULE)
+ const auto &ctbNonModMHpp = reinterpret_cast<CTBNonModule &>(buffer);
+ CHECK(ctbNonModMHpp.logicalName == "m.hpp" && ctbNonModMHpp.isHeaderUnit == false)
+
+ BTCNonModule nonModMPcm;
+ nonModMPcm.isHeaderUnit = false;
+ nonModMPcm.filePath = mHpp;
+ SEND_MESSAGE(nonModMPcm)
+ CHECK_RESULT(session.finish())
+ }
+
+ // compiling o.hpp
+ {
+ string compileCommand =
+ clangCommand + R"( -std=c++20 -fmodule-header=user -o ")" + oPcm + "\" -useIPC -xc++-header o.hpp";
+
+ auto managerResult = session.start(compileCommand, true);
+ CHECK_RESULT(managerResult)
+ auto &manager = *managerResult;
+
+ CHECK(type == CTB::NON_MODULE)
+ const auto &ctbNonModMHpp = reinterpret_cast<CTBNonModule &>(buffer);
+ CHECK(ctbNonModMHpp.logicalName == "m.hpp" && ctbNonModMHpp.isHeaderUnit == false)
+
+ BTCNonModule nonModMPcm;
+ nonModMPcm.isHeaderUnit = false;
+ nonModMPcm.filePath = mHpp;
+ SEND_MESSAGE(nonModMPcm)
+
+ CHECK_RESULT(session.readRequest())
+ CHECK(type == CTB::NON_MODULE)
+ const auto &ctbNonModNHpp = reinterpret_cast<CTBNonModule &>(buffer);
+ CHECK(ctbNonModNHpp.logicalName == "n.hpp" && ctbNonModNHpp.isHeaderUnit == true)
+
+ BTCNonModule nonModNPcm;
+ nonModNPcm.isHeaderUnit = true;
+
+ nonModNPcm.filePath = nPcm;
+
+ SEND_MESSAGE(nonModNPcm)
+ CHECK_RESULT(session.finish())
+ }
+
+ // compiling o.hpp with include-translation. BTCNonModule for n.hpp will be received with
+ // isHeaderUnit = true.
+ {
+ string compileCommand = clangCommand + R"( -std=c++20 -fmodule-header=user -o ")" + oPcm +
+ "\" -useIPC -xc++-header o.hpp -DTRANSLATING";
+
+ auto managerResult = session.start(compileCommand, true);
+ CHECK_RESULT(managerResult)
+ auto &manager = *managerResult;
+
+ CHECK(type == CTB::NON_MODULE)
+ const auto &ctbNonModMHpp = reinterpret_cast<CTBNonModule &>(buffer);
+ CHECK(ctbNonModMHpp.logicalName == "m.hpp" && ctbNonModMHpp.isHeaderUnit == false)
+
+ BTCNonModule nonModMPcm;
+ nonModMPcm.isHeaderUnit = false;
+ nonModMPcm.filePath = mHpp;
+ SEND_MESSAGE(nonModMPcm)
+
+ CHECK_RESULT(session.readRequest())
+ CHECK(type == CTB::NON_MODULE)
+ const auto &ctbNonModNHpp = reinterpret_cast<CTBNonModule &>(buffer);
+ CHECK(ctbNonModNHpp.logicalName == "n.hpp" && ctbNonModNHpp.isHeaderUnit == false)
+
+ BTCNonModule nonModNPcm;
+
+ nonModNPcm.isHeaderUnit = true;
+ nonModNPcm.filePath = nPcm;
+
+ SEND_MESSAGE(nonModNPcm)
+ CHECK_RESULT(session.finish())
+ }
+
+ // compiling big.hpp
+ {
+ string compileCommand =
+ clangCommand + R"( -std=c++20 -fmodule-header=user -o ")" + bigPcm + "\" -useIPC -xc++-header big.hpp";
+
+ auto managerResult = session.start(compileCommand, true);
+ CHECK_RESULT(managerResult)
+ auto &manager = *managerResult;
+
+ CHECK(type == CTB::NON_MODULE)
+ const auto &ctbNonModMHpp = reinterpret_cast<CTBNonModule &>(buffer);
+ CHECK(ctbNonModMHpp.logicalName == "x.hpp" && ctbNonModMHpp.isHeaderUnit == false)
+
+ BTCNonModule headerFile;
+ headerFile.isHeaderUnit = false;
+ headerFile.filePath = xHpp;
+ HeaderFile yHeaderFile;
+ yHeaderFile.logicalName = "y.hpp";
+ yHeaderFile.filePath = yHpp;
+ yHeaderFile.isSystem = true;
+ headerFile.headerFiles.emplace_back(yHeaderFile);
+ HeaderFile zHeaderFile;
+ zHeaderFile.logicalName = "z.hpp";
+ zHeaderFile.filePath = zHpp;
+ zHeaderFile.isSystem = true;
+ headerFile.headerFiles.emplace_back(zHeaderFile);
+
+ SEND_MESSAGE(headerFile)
+ CHECK_RESULT(session.finish())
+ }
+
+ // compiling foo.cpp
+ {
+ string compileCommand = clangCommand + R"( -std=c++20 -fmodules-reduced-bmi -o ")" + fooObj +
+ "\" -useIPC -c -xc++-module foo.cpp -fmodule-output=\"" + fooPcm + "\"";
+
+ auto managerResult = session.start(compileCommand, true);
+ CHECK_RESULT(managerResult)
+ auto &manager = *managerResult;
+
+ CHECK(type == CTB::NON_MODULE)
+ const auto &xHeader = reinterpret_cast<CTBNonModule &>(buffer);
+ CHECK(xHeader.logicalName == "x.hpp" && xHeader.isHeaderUnit == false)
+
+ BTCNonModule bigHu;
+ bigHu.isHeaderUnit = true;
+ bigHu.logicalNames.emplace_back("big.hpp");
+ bigHu.logicalNames.emplace_back("y.hpp");
+ bigHu.logicalNames.emplace_back("z.hpp");
+
+ bigHu.filePath = bigPcm;
+
+ SEND_MESSAGE(bigHu)
+
+ CHECK_RESULT(session.readRequest())
+ CHECK(type == CTB::MODULE)
+ const auto &aModule = reinterpret_cast<CTBModule &>(buffer);
+ CHECK(aModule.moduleName == "A")
+
+ BTCModule amod;
+ amod.filePath = aPcm;
+ ModuleDep abModDep;
+ abModDep.isHeaderUnit = false;
+ abModDep.filePath = aBPcm;
+ abModDep.logicalNames.emplace_back("A:B");
+ amod.modDeps.emplace_back(std::move(abModDep));
+ ModuleDep acModDep;
+ acModDep.isHeaderUnit = false;
+ acModDep.filePath = aCPcm;
+ acModDep.logicalNames.emplace_back("A:C");
+ amod.modDeps.emplace_back(std::move(acModDep));
+
+ SEND_MESSAGE(amod)
+ CHECK_RESULT(session.finish())
+ }
+
+ // compiling main.cpp
+ {
+ string compileCommand = clangCommand + R"( -std=c++20 -o ")" + mainObj + "\" -useIPC -c main.cpp";
+
+ auto managerResult = session.start(compileCommand, true);
+ CHECK_RESULT(managerResult)
+ auto &manager = *managerResult;
+
+ CHECK(type == CTB::MODULE)
+ const auto &ctbModule = reinterpret_cast<CTBModule &>(buffer);
+ CHECK(ctbModule.moduleName == "Foo")
+
+ BTCModule foo;
+ foo.filePath = fooPcm;
+
+ ModuleDep bigModDep;
+ bigModDep.isHeaderUnit = true;
+ bigModDep.filePath = bigPcm;
+ bigModDep.logicalNames.emplace_back("big.hpp");
+ bigModDep.logicalNames.emplace_back("x.hpp");
+ bigModDep.logicalNames.emplace_back("y.hpp");
+ bigModDep.logicalNames.emplace_back("z.hpp");
+ foo.modDeps.emplace_back(std::move(bigModDep));
+
+ ModuleDep aModDep;
+ aModDep.isHeaderUnit = false;
+ aModDep.filePath = aPcm;
+ aModDep.logicalNames.emplace_back("A");
+ foo.modDeps.emplace_back(std::move(aModDep));
+
+ ModuleDep bModDep;
+ bModDep.isHeaderUnit = false;
+ bModDep.filePath = aBPcm;
+ bModDep.logicalNames.emplace_back("A:B");
+ foo.modDeps.emplace_back(std::move(bModDep));
+
+ ModuleDep cModDep;
+ cModDep.isHeaderUnit = false;
+ cModDep.filePath = aCPcm;
+ cModDep.logicalNames.emplace_back("A:C");
+ foo.modDeps.emplace_back(std::move(cModDep));
+
+ SEND_MESSAGE(foo)
+
+ CHECK_RESULT(session.finish())
+ }
+
+ CHECK_RESULT(checkFailureCases(clangCommand, fooPcm, bigPcm))
+ CHECK_RESULT(checkHeaderUnitVisibility(clangCommand, aPcm, aBPcm, aCPcm))
+ fflush(stdout);
+ return {};
+}
+} // namespace
+
+#ifdef IS_THIS_CLANG_REPO
+TEST(IPC2978Test, IPC2978Test)
+{
+ std::error_code error;
+ const path originalDirectory = current_path(error);
+ ASSERT_FALSE(error) << error.message();
+ const path compiler = absolute(path(LLVM_TOOLS_BINARY_DIR) / clangExecutableName, error);
+ ASSERT_FALSE(error) << error.message();
+ llvm::SmallString<128> temporaryDirectory;
+ error = llvm::sys::fs::createUniqueDirectory("ipc2978-test", temporaryDirectory);
+ ASSERT_FALSE(error) << error.message();
+ llvm::scope_exit cleanup([&] {
+ current_path(originalDirectory, error);
+ EXPECT_FALSE(error) << error.message();
+ EXPECT_FALSE(llvm::sys::fs::remove_directories(temporaryDirectory));
+ });
+ current_path(path(temporaryDirectory.c_str()), error);
+ ASSERT_FALSE(error) << error.message();
+ if (const auto result = runTest(compiler); !result)
+ {
+ FAIL() << result.error();
+ }
+ EXPECT_TRUE(exists(path("main .o")));
+}
+#else
+int main(int argc, char **argv)
+{
+ if (argc > 2)
+ {
+ std::cerr << "Usage: " << argv[0] << " [clang-executable]\n";
+ return EXIT_FAILURE;
+ }
+ path compiler = argc == 2 ? argv[1] : IPC2978_CLANG_EXECUTABLE;
+ if (compiler.empty())
+ {
+ compiler = path(".") / clangExecutableName;
+ }
+ if (const auto &r = runTest(compiler); !r)
+ {
+ std::cout << r.error() << std::endl;
+ return EXIT_FAILURE;
+ }
+ if (!exists(path("main .o")))
+ {
+ std::cout << "main.o not found" << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}
+#endif
diff --git a/clang/unittests/IPC2978/TestBuildSystem.hpp b/clang/unittests/IPC2978/TestBuildSystem.hpp
new file mode 100644
index 00000000000000..3a675ccfc73f5b
--- /dev/null
+++ b/clang/unittests/IPC2978/TestBuildSystem.hpp
@@ -0,0 +1,61 @@
+// Response sending for the simulated build systems in the protocol and Clang tests.
+#ifndef IPC2978_TEST_BUILD_SYSTEM_HPP
+#define IPC2978_TEST_BUILD_SYSTEM_HPP
+
+#ifdef IS_THIS_CLANG_REPO
+#include "clang/IPC2978/Manager.hpp"
+#else
+#include "Manager.hpp"
+#endif
+#include <cstring>
+
+namespace ipc2978_test
+{
+class TestBuildSystem
+{
+ // Borrowed pipe descriptor on Unix, or HANDLE encoded as uint64_t on Windows.
+ uint64_t writeFd;
+
+ P2978::Result<void> writeInternal(std::string_view buffer) const
+ {
+#ifdef _WIN32
+ return P2978::Manager::writeAll(reinterpret_cast<void *>(writeFd), buffer);
+#else
+ return P2978::Manager::writeAll(static_cast<int>(writeFd), buffer.data(), buffer.size());
+#endif
+ }
+
+ public:
+ // The test process retains ownership of the compiler's input pipe.
+ explicit TestBuildSystem(uint64_t writeFd_) : writeFd(writeFd_)
+ {
+ }
+
+ [[nodiscard]] P2978::Result<void> sendMessage(const P2978::BTCModule &moduleFile) const
+ {
+ std::string buffer;
+ P2978::Manager::writePath(buffer, moduleFile.filePath);
+ buffer.push_back(moduleFile.isSystem);
+ P2978::Manager::writeVectorOfModuleDep(buffer, moduleFile.modDeps);
+ buffer.append(P2978::delimiter, std::strlen(P2978::delimiter));
+ return writeInternal(buffer);
+ }
+
+ [[nodiscard]] P2978::Result<void> sendMessage(const P2978::BTCNonModule &nonModule) const
+ {
+ std::string buffer;
+ buffer.push_back(nonModule.isHeaderUnit);
+ buffer.push_back(nonModule.isSystem);
+ P2978::Manager::writeVectorOfHeaderFiles(buffer, nonModule.headerFiles);
+ P2978::Manager::writePath(buffer, nonModule.filePath);
+ if (nonModule.isHeaderUnit)
+ {
+ P2978::Manager::writeVectorOfStrings(buffer, nonModule.logicalNames);
+ P2978::Manager::writeVectorOfHuDeps(buffer, nonModule.huDeps);
+ }
+ buffer.append(P2978::delimiter, std::strlen(P2978::delimiter));
+ return writeInternal(buffer);
+ }
+};
+} // namespace ipc2978_test
+#endif // IPC2978_TEST_BUILD_SYSTEM_HPP
diff --git a/clang/unittests/IPC2978/TestProcess.hpp b/clang/unittests/IPC2978/TestProcess.hpp
new file mode 100644
index 00000000000000..f10846af0c9747
--- /dev/null
+++ b/clang/unittests/IPC2978/TestProcess.hpp
@@ -0,0 +1,317 @@
+// Shared process and protocol support for the library and Clang integration tests.
+#ifndef IPC2978_TEST_PROCESS_HPP
+#define IPC2978_TEST_PROCESS_HPP
+
+#ifdef IS_THIS_CLANG_REPO
+#include "clang/IPC2978/IPCManagerBS.hpp"
+#else
+#include "IPCManagerBS.hpp"
+#endif
+#include <cstdio>
+#include <cstring>
+#include <string>
+#include <utility>
+#ifdef _WIN32
+#include <Windows.h>
+#else
+#include <cerrno>
+#include <csignal>
+#include <fcntl.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <wordexp.h>
+#endif
+
+namespace ipc2978_test
+{
+inline bool endsWith(const std::string &str, const std::string &suffix)
+{
+ return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+// Processes may run concurrently; a read waits for one selected child's next request or EOF.
+class TestProcess
+{
+ static constexpr uint64_t invalid = UINT64_MAX;
+ void (*onFailure)(const std::string &);
+
+ bool fail(std::string message)
+ {
+ error = std::move(message);
+ if (onFailure)
+ {
+ onFailure(error);
+ }
+ return false;
+ }
+
+ static void closePipe(uint64_t &pipe)
+ {
+ if (pipe == invalid)
+ {
+ return;
+ }
+#ifdef _WIN32
+ CloseHandle(reinterpret_cast<HANDLE>(pipe));
+#else
+ close(static_cast<int>(pipe));
+#endif
+ pipe = invalid;
+ }
+
+ public:
+ explicit TestProcess(void (*callback)(const std::string &) = nullptr) : onFailure(callback)
+ {
+ }
+ TestProcess(const TestProcess &) = delete;
+ TestProcess &operator=(const TestProcess &) = delete;
+
+ ~TestProcess()
+ {
+ // An assertion may stop the test while the compiler is waiting for a response.
+ if (pid != invalid)
+ {
+#ifdef _WIN32
+ TerminateProcess(reinterpret_cast<HANDLE>(pid), 1);
+ WaitForSingleObject(reinterpret_cast<HANDLE>(pid), INFINITE);
+ CloseHandle(reinterpret_cast<HANDLE>(pid));
+#else
+ kill(static_cast<pid_t>(pid), SIGKILL);
+ while (waitpid(static_cast<pid_t>(pid), nullptr, 0) == -1 && errno == EINTR)
+ {
+ }
+#endif
+ }
+ closePipe(readPipe);
+ closePipe(writePipe);
+ }
+
+ uint64_t pid = invalid;
+ uint64_t readPipe = invalid;
+ uint64_t writePipe = invalid;
+ int exitStatus = -1;
+ std::string error;
+
+ bool startAsyncProcess(const char *command)
+ {
+ if (pid != invalid)
+ {
+ return fail("A test process is already running");
+ }
+ error.clear();
+ exitStatus = -1;
+#ifdef _WIN32
+ SECURITY_ATTRIBUTES attributes{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE};
+ HANDLE outputRead = nullptr, outputWrite = nullptr;
+ HANDLE inputRead = nullptr, inputWrite = nullptr;
+ if (!CreatePipe(&outputRead, &outputWrite, &attributes, 0))
+ {
+ return fail(P2978::getErrorString());
+ }
+ if (!CreatePipe(&inputRead, &inputWrite, &attributes, 0))
+ {
+ const auto message = P2978::getErrorString();
+ CloseHandle(outputRead);
+ CloseHandle(outputWrite);
+ return fail(message);
+ }
+ readPipe = reinterpret_cast<uint64_t>(outputRead);
+ writePipe = reinterpret_cast<uint64_t>(inputWrite);
+ if (!SetHandleInformation(outputRead, HANDLE_FLAG_INHERIT, 0) ||
+ !SetHandleInformation(inputWrite, HANDLE_FLAG_INHERIT, 0))
+ {
+ const auto message = P2978::getErrorString();
+ CloseHandle(inputRead);
+ CloseHandle(outputWrite);
+ closePipe(readPipe);
+ closePipe(writePipe);
+ return fail(message);
+ }
+ STARTUPINFOA startup{};
+ startup.cb = sizeof(startup);
+ startup.dwFlags = STARTF_USESTDHANDLES;
+ startup.hStdInput = inputRead;
+ startup.hStdOutput = outputWrite;
+ startup.hStdError = outputWrite;
+ PROCESS_INFORMATION process{};
+ std::string mutableCommand(command);
+ const bool started = CreateProcessA(nullptr, mutableCommand.data(), nullptr, nullptr, TRUE,
+ CREATE_NEW_PROCESS_GROUP, nullptr, nullptr, &startup, &process);
+ const auto message = started ? std::string() : P2978::getErrorString();
+ CloseHandle(inputRead);
+ CloseHandle(outputWrite);
+ if (!started)
+ {
+ closePipe(readPipe);
+ closePipe(writePipe);
+ return fail(message);
+ }
+ CloseHandle(process.hThread);
+ pid = reinterpret_cast<uint64_t>(process.hProcess);
+#else
+ int output[2], input[2];
+ if (pipe(output) == -1)
+ {
+ return fail(P2978::getErrorString());
+ }
+ if (pipe(input) == -1)
+ {
+ const auto message = P2978::getErrorString();
+ close(output[0]);
+ close(output[1]);
+ return fail(message);
+ }
+ // A later child must not inherit the pipe ends of an earlier child.
+ for (int fd : {output[0], output[1], input[0], input[1]})
+ {
+ if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
+ {
+ const auto message = P2978::getErrorString();
+ for (int pipeFd : {output[0], output[1], input[0], input[1]})
+ {
+ close(pipeFd);
+ }
+ return fail(message);
+ }
+ }
+ const pid_t child = fork();
+ if (child == 0)
+ {
+ if (dup2(input[0], STDIN_FILENO) == -1 || dup2(output[1], STDOUT_FILENO) == -1 ||
+ dup2(output[1], STDERR_FILENO) == -1)
+ {
+ _exit(127);
+ }
+ for (int fd : {output[0], output[1], input[0], input[1]})
+ {
+ close(fd);
+ }
+ wordexp_t words{};
+ if (wordexp(command, &words, WRDE_NOCMD) != 0 || words.we_wordc == 0)
+ {
+ _exit(127);
+ }
+ execvp(words.we_wordv[0], words.we_wordv);
+ perror("execvp");
+ _exit(127);
+ }
+ const auto message = child == -1 ? P2978::getErrorString() : std::string();
+ close(output[1]);
+ close(input[0]);
+ readPipe = output[0];
+ writePipe = input[1];
+ if (child == -1)
+ {
+ closePipe(readPipe);
+ closePipe(writePipe);
+ return fail(message);
+ }
+ pid = child;
+#endif
+ return true;
+ }
+
+ bool reapProcess()
+ {
+ if (pid == invalid)
+ {
+ return fail("No test process to reap");
+ }
+#ifdef _WIN32
+ if (WaitForSingleObject(reinterpret_cast<HANDLE>(pid), INFINITE) != WAIT_OBJECT_0)
+ {
+ return fail(P2978::getErrorString());
+ }
+ DWORD status = 0;
+ if (!GetExitCodeProcess(reinterpret_cast<HANDLE>(pid), &status))
+ {
+ return fail(P2978::getErrorString());
+ }
+ exitStatus = static_cast<int>(status);
+ CloseHandle(reinterpret_cast<HANDLE>(pid));
+#else
+ int status;
+ pid_t result;
+ do
+ {
+ result = waitpid(static_cast<pid_t>(pid), &status, 0);
+ } while (result == -1 && errno == EINTR);
+ if (result == -1)
+ {
+ return fail(P2978::getErrorString());
+ }
+ exitStatus = WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
+#endif
+ pid = invalid;
+ closePipe(readPipe);
+ closePipe(writePipe);
+ return true;
+ }
+
+ // True means a complete protocol frame. False means EOF or an error recorded in error.
+ bool readCompilerMessage(std::string &output)
+ {
+ if (readPipe == invalid)
+ {
+ return fail("No test process output pipe");
+ }
+ while (true)
+ {
+ char buffer[4096];
+#ifdef _WIN32
+ DWORD count = 0;
+ if (!ReadFile(reinterpret_cast<HANDLE>(readPipe), buffer, sizeof(buffer), &count, nullptr))
+ {
+ if (GetLastError() == ERROR_BROKEN_PIPE)
+ {
+ return false;
+ }
+ return fail(P2978::getErrorString());
+ }
+#else
+ const uint64_t count = read(static_cast<int>(readPipe), buffer, sizeof(buffer));
+ if (count == UINT64_MAX)
+ {
+ if (errno == EINTR)
+ {
+ continue;
+ }
+ return fail(P2978::getErrorString());
+ }
+#endif
+ if (count == 0)
+ {
+ return false;
+ }
+ output.append(buffer, count);
+ if (endsWith(output, P2978::delimiter))
+ {
+ return true;
+ }
+ }
+ }
+
+ bool pruneCompilerOutput(std::string &output, char (&buffer)[320], P2978::CTB &type)
+ {
+ const uint64_t trailerSize = sizeof(uint32_t) + strlen(P2978::delimiter);
+ if (output.size() < trailerSize)
+ {
+ return fail("Received IPC frame without a payload size");
+ }
+ uint32_t payloadSize;
+ memcpy(&payloadSize, output.data() + output.size() - trailerSize, sizeof(payloadSize));
+ if (payloadSize > output.size() - trailerSize)
+ {
+ return fail("Received IPC payload size exceeds the available bytes");
+ }
+ const char *payload = output.data() + output.size() - trailerSize - payloadSize;
+ if (const auto result = P2978::IPCManagerBS::receiveMessage(buffer, type, {payload, payloadSize}); !result)
+ {
+ return fail(result.error());
+ }
+ output.resize(output.size() - trailerSize - payloadSize);
+ return true;
+ }
+};
+} // namespace ipc2978_test
+#endif
More information about the cfe-commits
mailing list