[clang] [Feature]: merge host and kernel dependencies for heterogeneous compilation (PR #119513)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Feb 18 02:56:45 PST 2025
https://github.com/zhouronghua updated https://github.com/llvm/llvm-project/pull/119513
>From 897cb52dcd18f9ada4bd39e17f3c411654b5f178 Mon Sep 17 00:00:00 2001
From: "ronghua.zhou" <ronghua.zhou at enflame-tech.com>
Date: Fri, 14 Feb 2025 01:04:51 +0000
Subject: [PATCH] [Feature]: support for the BC library file into the compile
dependencies
---
clang/include/clang/Driver/Options.td | 2 +
clang/lib/Driver/ToolChains/Clang.cpp | 16 +++-
clang/lib/Frontend/CompilerInstance.cpp | 10 ++-
clang/lib/Frontend/DependencyFile.cpp | 107 ++++++++++++++++++++++--
4 files changed, 125 insertions(+), 10 deletions(-)
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 5ad187926e710..12b54010753df 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -798,6 +798,8 @@ def MD : Flag<["-"], "MD">, Group<M_Group>,
HelpText<"Write a depfile containing user and system headers">;
def MMD : Flag<["-"], "MMD">, Group<M_Group>,
HelpText<"Write a depfile containing user headers">;
+def MMMD : Flag<["-"], "MMMD">, Group<M_Group>,
+ HelpText<"Merge multiple depfile if available">;
def M : Flag<["-"], "M">, Group<M_Group>,
HelpText<"Like -MD, but also implies -E and writes to stdout by default">;
def MM : Flag<["-"], "MM">, Group<M_Group>,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 55ec3db0ee994..6c60ea1c96fba 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1056,8 +1056,20 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
DepFile = getDependencyFileName(Args, Inputs);
C.addFailureResultFile(DepFile, &JA);
}
- CmdArgs.push_back("-dependency-file");
- CmdArgs.push_back(DepFile);
+ // for host compile, if OPT_MMMD is on, default to create *.d.host file
+ // and merge it with *.d (kernel dep) file in DependencyFile.cpp
+ Arg *ArgMMD = Args.getLastArg(options::OPT_MMMD);
+ auto at = getToolChain().getAuxTriple();
+ if (ArgMMD && !at && std::string(DepFile) != "-") {
+ SmallString<128> NewDepFile(DepFile);
+ NewDepFile.append(".host");
+ CmdArgs.push_back("-dependency-file");
+ CmdArgs.push_back(Args.MakeArgString(NewDepFile));
+ // else keep the original dep file name
+ } else {
+ CmdArgs.push_back("-dependency-file");
+ CmdArgs.push_back(DepFile);
+ }
}
bool HasTarget = false;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index c11c857ea0606..60ac343391e18 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -494,8 +494,14 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
// Handle generating dependencies, if requested.
const DependencyOutputOptions &DepOpts = getDependencyOutputOpts();
- if (!DepOpts.OutputFile.empty())
- addDependencyCollector(std::make_shared<DependencyFileGenerator>(DepOpts));
+ if (!DepOpts.OutputFile.empty()) {
+ auto DFG = std::make_shared<DependencyFileGenerator>(DepOpts);
+ for (auto F : getCodeGenOpts().LinkBitcodeFiles) {
+ DFG->maybeAddDependency(F.Filename, false, false, false, false);
+ }
+ addDependencyCollector(DFG);
+ }
+
if (!DepOpts.DOTOutputFile.empty())
AttachDependencyGraphGen(*PP, DepOpts.DOTOutputFile,
getHeaderSearchOpts().Sysroot);
diff --git a/clang/lib/Frontend/DependencyFile.cpp b/clang/lib/Frontend/DependencyFile.cpp
index 15fa7de35df97..5e63e42f72c04 100644
--- a/clang/lib/Frontend/DependencyFile.cpp
+++ b/clang/lib/Frontend/DependencyFile.cpp
@@ -10,6 +10,9 @@
//
//===----------------------------------------------------------------------===//
+#include <set>
+#include <string>
+
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Frontend/DependencyOutputOptions.h"
@@ -350,13 +353,105 @@ void DependencyFileGenerator::outputDependencyFile(DiagnosticsEngine &Diags) {
}
std::error_code EC;
- llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_TextWithCRLF);
- if (EC) {
- Diags.Report(diag::err_fe_error_opening) << OutputFile << EC.message();
- return;
- }
- outputDependencyFile(OS);
+ // merge host dependency file (*.d.host)
+ // to kernel dependency file (*.d.host) for tops target
+ llvm::StringRef SubStr = ".host";
+ SmallString<128> OutputFileS(OutputFile);
+ size_t Pos = OutputFileS.find(SubStr);
+ // for tops target, trim .host in dep file
+ if (Pos != llvm::StringRef::npos) {
+ auto ndf = OutputFileS.substr(0, Pos);
+ // dependencies is a set to auto merge duplicate dependencies
+ std::set<std::string> dependencies;
+
+ std::string line;
+ std::string tmpline;
+ std::string KernelStartLine;
+ std::string HostStartLine;
+ std::string Endline;
+
+ // Read kernel dep file
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> KernelDepFile =
+ llvm::MemoryBuffer::getFile(ndf.str());
+ if (KernelDepFile) {
+ llvm::StringRef KernelDepContent = KernelDepFile.get()->getBuffer();
+ for (const auto &tmpline : llvm::split(KernelDepContent, '\n')) {
+ // Remove empty lines and comment lines
+ if (!tmpline.empty() && tmpline[0] != '#') {
+ line = tmpline.str();
+ if (KernelStartLine.empty()) {
+ KernelStartLine = line;
+ } else {
+ dependencies.insert(line);
+ }
+ }
+ }
+ // Process Endline
+ Endline = line;
+ dependencies.erase(Endline); // Remove Endline from dependencies
+ }
+
+ // Read host dep file
+ std::string HostDep;
+ llvm::raw_string_ostream OSS(HostDep);
+ outputDependencyFile(OSS);
+ if (!HostDep.empty()) {
+ for (const auto &tmpline : llvm::split(HostDep, '\n')) {
+ // Remove empty lines and comment lines
+ if (!tmpline.empty() && tmpline[0] != '#') {
+ line = tmpline.str();
+ if (HostStartLine.empty()) {
+ HostStartLine = line;
+ // if KernelStartLine is not empty
+ if (!KernelStartLine.empty() && HostStartLine != KernelStartLine) {
+ Diags.Report(diag::err_fe_error_opening)
+ << OutputFile << "host dep file is not match kernel dep file";
+ return;
+ }
+ } else {
+ dependencies.insert(line);
+ }
+ }
+ }
+ // Process Endline
+ dependencies.erase(line);
+ if (!Endline.empty()) {
+ if (line != Endline) {
+ dependencies.insert(line + " \\");
+ }
+ } else {
+ Endline = line;
+ }
+ }
+
+ // Write merged dep file
+ llvm::raw_fd_ostream DepFile(ndf.str(), EC, llvm::sys::fs::OF_Text);
+ if (EC) {
+ Diags.Report(diag::err_fe_error_opening) << ndf.str() << EC.message();
+ return;
+ }
+ // Write HostStartLine, KernelStartLine maybe is empty.
+ if (!HostStartLine.empty()) {
+ DepFile << HostStartLine << "\n";
+ }
+ for (const auto &dep : dependencies) {
+ DepFile << dep << "\n";
+ }
+ // Write Endline
+ if (!Endline.empty()) {
+ DepFile << Endline << "\n";
+ }
+ } else {
+
+ llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_Text);
+ if (EC) {
+ Diags.Report(diag::err_fe_error_opening) << OutputFile << EC.message();
+ return;
+ }
+
+ outputDependencyFile(OS);
+ }
}
void DependencyFileGenerator::outputDependencyFile(llvm::raw_ostream &OS) {
More information about the cfe-commits
mailing list