[clang] [Feature]: merge host and kernel dependencies for heterogeneous compilation (PR #119513)

[via cfe-commits]

https://github.com/zhouronghua updated https://github.com/llvm/llvm-project/pull/119513

>From 9fed66b422dc74f8feb9708c7186ebc25a56f8b9 Mon Sep 17 00:00:00 2001
From: "ronghua.zhou" <ronghua.zhou at enflame-tech.com>
Date: Fri, 14 Feb 2025 01:04:51 +0000
Subject: [PATCH] [Feature]: support for the BC library file into the compile
 dependencies

---
 clang/lib/Driver/ToolChains/Clang.cpp   | 15 +++-
 clang/lib/Frontend/CompilerInstance.cpp | 10 ++-
 clang/lib/Frontend/DependencyFile.cpp   | 91 +++++++++++++++++++++++++
 3 files changed, 112 insertions(+), 4 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 55ec3db0ee994..1402898b1cd89 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1056,8 +1056,19 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
         DepFile = getDependencyFileName(Args, Inputs);
         C.addFailureResultFile(DepFile, &JA);
       }
-      CmdArgs.push_back("-dependency-file");
-      CmdArgs.push_back(DepFile);
+      // for host compile, default to create *.d.host file
+      // and merge it with *.d (kernel dep) file in DependencyFile.cpp
+      auto at = getToolChain().getAuxTriple();
+      if (!at && std::string(DepFile) != "-") {
+        SmallString<128> NewDepFile(DepFile);
+        NewDepFile.append(".host");
+        CmdArgs.push_back("-dependency-file");
+        CmdArgs.push_back(Args.MakeArgString(NewDepFile));
+        // else keep the original dep file name
+      } else {
+        CmdArgs.push_back("-dependency-file");
+        CmdArgs.push_back(DepFile);
+      }
     }
 
     bool HasTarget = false;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index c11c857ea0606..60ac343391e18 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -494,8 +494,14 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
 
   // Handle generating dependencies, if requested.
   const DependencyOutputOptions &DepOpts = getDependencyOutputOpts();
-  if (!DepOpts.OutputFile.empty())
-    addDependencyCollector(std::make_shared<DependencyFileGenerator>(DepOpts));
+  if (!DepOpts.OutputFile.empty()) {
+    auto DFG = std::make_shared<DependencyFileGenerator>(DepOpts);
+    for (auto F : getCodeGenOpts().LinkBitcodeFiles) {
+      DFG->maybeAddDependency(F.Filename, false, false, false, false);
+    }
+    addDependencyCollector(DFG);
+  }
+
   if (!DepOpts.DOTOutputFile.empty())
     AttachDependencyGraphGen(*PP, DepOpts.DOTOutputFile,
                              getHeaderSearchOpts().Sysroot);
diff --git a/clang/lib/Frontend/DependencyFile.cpp b/clang/lib/Frontend/DependencyFile.cpp
index 15fa7de35df97..ce612d8522ff8 100644
--- a/clang/lib/Frontend/DependencyFile.cpp
+++ b/clang/lib/Frontend/DependencyFile.cpp
@@ -10,6 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <iostream>
+#include <fstream>
+#include <set>
+#include <string>
+#include <vector>
+
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/DependencyOutputOptions.h"
@@ -357,6 +363,91 @@ void DependencyFileGenerator::outputDependencyFile(DiagnosticsEngine &Diags) {
   }
 
   outputDependencyFile(OS);
+  OS.flush();
+  OS.close();
+
+  // merge host dependency file (*.d.host)
+  // to kernel dependency file (*.d.host) for tops target
+  llvm::StringRef SubStr = ".host";
+  SmallString<128> OutputFileS(OutputFile);
+  size_t Pos = OutputFileS.find(SubStr);
+  // for tops target, trim .host in dep file
+  if (Pos != llvm::StringRef::npos) {
+    auto ndf = OutputFileS.substr(0, Pos);
+    // dependencies is a set to auto merge duplicate dependencies
+    std::set<std::string> dependencies;
+
+    std::string line;
+    std::string tmpline;
+    std::string KernelStartLine;
+    std::string HostStartLine;
+    std::string Endline;
+    // read kernel dep file
+    std::ifstream KernelDepFile(ndf.str());
+    if (KernelDepFile.is_open()) {
+      while (std::getline(KernelDepFile, tmpline)) {
+        // Remove empty lines and comment lines
+        if (!tmpline.empty() && tmpline[0] != '#') {
+          line = tmpline;
+          if (KernelStartLine.empty()) {
+            KernelStartLine = line;
+          } else {
+            dependencies.insert(line);
+          }
+        }
+      }
+      // Process Endline
+      Endline = line;
+      dependencies.erase(line);
+    }
+    // read host dep file
+    std::ifstream HostDepFile(OutputFile);
+    if (HostDepFile.is_open()) {
+      while (std::getline(HostDepFile, tmpline)) {
+        // Remove empty lines and comment lines
+        if (!tmpline.empty() && tmpline[0] != '#') {
+          line = tmpline;
+          if (HostStartLine.empty()) {
+            HostStartLine = line;
+            // if KernelStartLine is not empty
+            if (KernelStartLine.length() > 0) {
+              if (HostStartLine != KernelStartLine) {
+                Diags.Report(diag::err_fe_error_opening)
+                    << OutputFile
+                    << "host dep file is not match kernel dep file";
+                return;
+              }
+            }
+          } else {
+            dependencies.insert(line);
+          }
+        }
+      }
+      // Process Endline
+      dependencies.erase(line);
+
+      if (Endline.length() > 0) {
+        if (line != Endline) {
+          dependencies.insert(line + " \\");
+        }
+      } else {
+        Endline = line;
+      }
+    }
+    // write merged dep file
+    std::ofstream DepFile(ndf.str());
+    if (DepFile.is_open()) {
+      // Write HostStartLine, KernelStartLine maybe is empty.
+      DepFile << HostStartLine<< std::endl;
+      for (const auto &dep : dependencies) {
+        DepFile << dep << std::endl;
+      }
+      // Write Endline
+      DepFile << Endline<< std::endl;
+    }
+    // unlink host dep file
+    llvm::sys::fs::remove(OutputFile);
+  }
 }
 
 void DependencyFileGenerator::outputDependencyFile(llvm::raw_ostream &OS) {