[clang] [Feature]: merge host and kernel dependencies for heterogeneous compilation (PR #119513)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Mar 5 23:10:51 PST 2025
https://github.com/zhouronghua updated https://github.com/llvm/llvm-project/pull/119513
>From b8a0f37d4d408be7877c8134ff4654c6289e7e21 Mon Sep 17 00:00:00 2001
From: "ronghua.zhou" <ronghua.zhou at enflame-tech.com>
Date: Fri, 14 Feb 2025 01:04:51 +0000
Subject: [PATCH] [Feature]: support for the BC library file into the compile
dependencies
---
clang/lib/Driver/ToolChains/Clang.cpp | 19 +++-
clang/lib/Frontend/CompilerInstance.cpp | 10 ++-
clang/lib/Frontend/DependencyFile.cpp | 114 ++++++++++++++++++++++--
3 files changed, 133 insertions(+), 10 deletions(-)
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 55ec3db0ee994..e27c1fe95af42 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1056,8 +1056,23 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
DepFile = getDependencyFileName(Args, Inputs);
C.addFailureResultFile(DepFile, &JA);
}
- CmdArgs.push_back("-dependency-file");
- CmdArgs.push_back(DepFile);
+ // for host compile, we changed the dep file name to *.d.CUID.host
+ // so it will not overide kernel dep file,
+ // and merge it with *.d (kernel dep) file in DependencyFile.cpp
+ // for example, abc.d -> abc.d.2282B80C.host
+ auto AT = getToolChain().getAuxTriple();
+ if (!AT && std::string(DepFile) != "-") {
+ SmallString<128> NewDepFile(DepFile);
+ NewDepFile.append(
+ "." + llvm::utohexstr(llvm::sys::Process::GetRandomNumber()) +
+ ".host");
+ CmdArgs.push_back("-dependency-file");
+ CmdArgs.push_back(Args.MakeArgString(NewDepFile));
+ // else keep the original dep file name
+ } else {
+ CmdArgs.push_back("-dependency-file");
+ CmdArgs.push_back(DepFile);
+ }
}
bool HasTarget = false;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index c11c857ea0606..60ac343391e18 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -494,8 +494,14 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
// Handle generating dependencies, if requested.
const DependencyOutputOptions &DepOpts = getDependencyOutputOpts();
- if (!DepOpts.OutputFile.empty())
- addDependencyCollector(std::make_shared<DependencyFileGenerator>(DepOpts));
+ if (!DepOpts.OutputFile.empty()) {
+ auto DFG = std::make_shared<DependencyFileGenerator>(DepOpts);
+ for (auto F : getCodeGenOpts().LinkBitcodeFiles) {
+ DFG->maybeAddDependency(F.Filename, false, false, false, false);
+ }
+ addDependencyCollector(DFG);
+ }
+
if (!DepOpts.DOTOutputFile.empty())
AttachDependencyGraphGen(*PP, DepOpts.DOTOutputFile,
getHeaderSearchOpts().Sysroot);
diff --git a/clang/lib/Frontend/DependencyFile.cpp b/clang/lib/Frontend/DependencyFile.cpp
index 15fa7de35df97..9b3955e1d333e 100644
--- a/clang/lib/Frontend/DependencyFile.cpp
+++ b/clang/lib/Frontend/DependencyFile.cpp
@@ -10,6 +10,9 @@
//
//===----------------------------------------------------------------------===//
+#include <set>
+#include <string>
+
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Frontend/DependencyOutputOptions.h"
@@ -343,6 +346,79 @@ static void PrintFilename(raw_ostream &OS, StringRef Filename,
}
}
+static std::vector<std::string> SplitLines(llvm::StringRef &Dep) {
+ std::vector<std::string> Deps;
+
+ for (const auto &line : llvm::split(KernelDepContent, '\n'))
+ // Remove empty lines and comment lines
+ if (!tmpline.empty() && tmpline[0] != '#')
+ Deps.insert(line);
+
+ return Deps;
+}
+
+static std::string GetKernelDepFileName(std::string &HostDepFileName) {
+
+ // merge host dependency file (*.d.host)
+ // to kernel dependency file (*.d.host) for tops target
+ // for example, abc.d -> abc.d.2282B80C.host
+ const int CUIDLEN = 9;
+ llvm::StringRef SubStr = ".host";
+ SmallString<128> OutputFileS(HostDepFileName);
+ size_t Pos = OutputFileS.find(SubStr);
+ // for tops target, trim .host in dep file
+ if (Pos != llvm::StringRef::npos)
+ // abc.d.2282B80C.host -> abc.d
+ return OutputFileS.substr(0, Pos - CUIDLEN);
+ else
+ return "";
+}
+
+static std::set<std::string>
+TryMergeDependencyFile(std::vector<std::string> &KD,
+ std::vector<std::string> &HD,
+ std::string &KDFN) {
+ if (HD.empty() && KD.empty())
+ // both kernel and host dep file is empty
+ return;
+
+ // Write merged dep file
+ llvm::raw_fd_ostream DF(KDFN, EC, llvm::sys::fs::OF_Text);
+ if (EC) {
+ Diags.Report(diag::err_fe_error_opening) << KDFN << EC.message();
+ return;
+ }
+ if (HD.empty())
+ for (const auto &DL : KD)
+ DF << DL << "\n";
+ else if (KD.empty())
+ for (const auto &DL : HD)
+ DF << DL << "\n";
+ else {
+ if (KD.front() != HD.front())
+ Diags.Report(diag::err_fe_error_opening)
+ << OutputFile << "host dep file is not match kernel dep file";
+ else {
+ // Write first line
+ DF << KD.front() << "\n";
+ // merge kernel and host dep file except first and last line
+ std::set<std::string> D;
+ D.insert(KD.begen() +1, KD.end()-1);
+ D.insert(HD.begen() +1, HD.end()-1);
+ DF
+ for (const auto &DL : D)
+ DF << DL << "\n";
+ // Write last line
+ if(KD.back() != HD.back()) {
+ DF << KD.back() << "\\\n";
+ DF << HD.back() << "\n";
+ } else {
+ DF << KD.back() << "\n";
+ }
+ }
+ }
+}
+
void DependencyFileGenerator::outputDependencyFile(DiagnosticsEngine &Diags) {
if (SeenMissingHeader) {
llvm::sys::fs::remove(OutputFile);
@@ -350,13 +426,39 @@ void DependencyFileGenerator::outputDependencyFile(DiagnosticsEngine &Diags) {
}
std::error_code EC;
- llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_TextWithCRLF);
- if (EC) {
- Diags.Report(diag::err_fe_error_opening) << OutputFile << EC.message();
- return;
- }
- outputDependencyFile(OS);
+ std::string KDFN = GetKernelDepFileName(OutputFile);
+ // if need to merge kernel and host dep file
+ if (KDFN != "") {
+ std::vector<std::string> KD;
+ std::vector<std::string> HD;
+
+ // Read kernel dep file
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> KDF =
+ llvm::MemoryBuffer::getFile(KDFN);
+ if (KDF) {
+ llvm::StringRef KDC = KDF.get()->getBuffer();
+ KD = SplitLines(KDC);
+ }
+
+ // Read host dep file
+ std::string HDC;
+ llvm::raw_string_ostream OSS(HDC);
+ outputDependencyFile(OSS);
+ OSS.flush();
+ if (!HDC.empty())
+ HD = SplitLines(HDC);
+ TryMergeDependencyFile(KD, HD, KDFN);
+ } else {
+ // merge is not needed, just write the dep file
+ llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_Text);
+ if (EC) {
+ Diags.Report(diag::err_fe_error_opening) << OutputFile << EC.message();
+ return;
+ }
+
+ outputDependencyFile(OS);
+ }
}
void DependencyFileGenerator::outputDependencyFile(llvm::raw_ostream &OS) {
More information about the cfe-commits
mailing list