[clang] [Feature]: merge host and kernel dependencies for heterogeneous compilation (PR #119513)
via cfe-commits
cfe-commits at lists.llvm.org
Fri Mar 7 02:38:24 PST 2025
https://github.com/zhouronghua updated https://github.com/llvm/llvm-project/pull/119513
>From 012ddd00153cd4528a5573a69cff640c75b34dbd Mon Sep 17 00:00:00 2001
From: "ronghua.zhou" <ronghua.zhou at enflame-tech.com>
Date: Fri, 14 Feb 2025 01:04:51 +0000
Subject: [PATCH] [Feature]: support for the BC library file into the compile
dependencies
---
clang/lib/Driver/ToolChains/Clang.cpp | 19 ++-
clang/lib/Frontend/CompilerInstance.cpp | 10 +-
clang/lib/Frontend/DependencyFile.cpp | 115 +++++++++++++++++-
clang/test/ClangScanDeps/P1689.cppm | 6 +-
clang/test/Driver/Wp-args.c | 4 +-
clang/test/Driver/cl-options.c | 2 +-
...-file-flag-with-multiple-offload-archs.hip | 8 +-
clang/test/Driver/m-and-mm.c | 12 +-
8 files changed, 150 insertions(+), 26 deletions(-)
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 55ec3db0ee994..e27c1fe95af42 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1056,8 +1056,23 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
DepFile = getDependencyFileName(Args, Inputs);
C.addFailureResultFile(DepFile, &JA);
}
- CmdArgs.push_back("-dependency-file");
- CmdArgs.push_back(DepFile);
+ // for host compile, we changed the dep file name to *.d.CUID.host
+ // so it will not overide kernel dep file,
+ // and merge it with *.d (kernel dep) file in DependencyFile.cpp
+ // for example, abc.d -> abc.d.2282B80C.host
+ auto AT = getToolChain().getAuxTriple();
+ if (!AT && std::string(DepFile) != "-") {
+ SmallString<128> NewDepFile(DepFile);
+ NewDepFile.append(
+ "." + llvm::utohexstr(llvm::sys::Process::GetRandomNumber()) +
+ ".host");
+ CmdArgs.push_back("-dependency-file");
+ CmdArgs.push_back(Args.MakeArgString(NewDepFile));
+ // else keep the original dep file name
+ } else {
+ CmdArgs.push_back("-dependency-file");
+ CmdArgs.push_back(DepFile);
+ }
}
bool HasTarget = false;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index c11c857ea0606..60ac343391e18 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -494,8 +494,14 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
// Handle generating dependencies, if requested.
const DependencyOutputOptions &DepOpts = getDependencyOutputOpts();
- if (!DepOpts.OutputFile.empty())
- addDependencyCollector(std::make_shared<DependencyFileGenerator>(DepOpts));
+ if (!DepOpts.OutputFile.empty()) {
+ auto DFG = std::make_shared<DependencyFileGenerator>(DepOpts);
+ for (auto F : getCodeGenOpts().LinkBitcodeFiles) {
+ DFG->maybeAddDependency(F.Filename, false, false, false, false);
+ }
+ addDependencyCollector(DFG);
+ }
+
if (!DepOpts.DOTOutputFile.empty())
AttachDependencyGraphGen(*PP, DepOpts.DOTOutputFile,
getHeaderSearchOpts().Sysroot);
diff --git a/clang/lib/Frontend/DependencyFile.cpp b/clang/lib/Frontend/DependencyFile.cpp
index 15fa7de35df97..93f9b38c82d54 100644
--- a/clang/lib/Frontend/DependencyFile.cpp
+++ b/clang/lib/Frontend/DependencyFile.cpp
@@ -10,6 +10,11 @@
//
//===----------------------------------------------------------------------===//
+#include <set>
+#include <string>
+#include <vector>
+#include <algorithm>
+
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Frontend/DependencyOutputOptions.h"
@@ -343,20 +348,118 @@ static void PrintFilename(raw_ostream &OS, StringRef Filename,
}
}
+static std::vector<std::string> SplitToLines(llvm::StringRef &Dep) {
+ std::vector<std::string> Deps;
+
+ for (const auto &line : llvm::split(Dep, '\n'))
+ // Remove empty lines and comment lines
+ if (!line.empty() && line[0] != '#')
+ Deps.push_back(line.str());
+
+ return Deps;
+}
+
+static std::string GetKernelDepFileName(std::string &HostDepFileName) {
+
+ // merge host dependency file (*.d.CUID.host)
+ // to kernel dependency file (*.d) for tops target
+ // for example, abc.d -> abc.d.2282B80C.host
+ const int CUIDLEN = 9;
+ llvm::StringRef SubStr = ".host";
+ SmallString<128> OutputFileS(HostDepFileName);
+ size_t Pos = OutputFileS.find(SubStr);
+ // for tops target, trim .CUID.host in dep file name
+ if (Pos != llvm::StringRef::npos)
+ // abc.d.2282B80C.host -> abc.d
+ return std::string(OutputFileS.substr(0, Pos - CUIDLEN));
+ else
+ return "";
+}
+
+static void TryMergeDependencyFile(std::vector<std::string> &KD,
+ std::vector<std::string> &HD,
+ llvm::raw_fd_ostream &DF,
+ DiagnosticsEngine &Diags) {
+ std::error_code EC;
+ // both kernel and host dep file must not be empty
+ assert(!HD.empty() && !KD.empty());
+
+ // if object file name is different, maybe comes from two test
+ // cases, just write host dep file to merged dep file
+ if (KD.front() != HD.front())
+ for (const auto &DL : HD)
+ DF << DL << "\n";
+ else {
+ // Write first line, which is the object file name
+ DF << KD.front() << "\n";
+ // add a splash at the end of each last line
+ KD.back() = KD.back() + " \\";
+ HD.back() = HD.back() + " \\";
+ // merge kernel and host dep file except first line
+ std::vector<std::string> D(KD.size() - 1 + HD.size() - 1);
+ auto E = std::set_union(KD.begin() + 1, KD.end(), HD.begin() + 1, HD.end(),
+ D.begin());
+ D.resize(E - D.begin());
+ // remove the redundent splash
+ D.back() = D.back().substr(0, D.back().size() - 2);
+ for (const auto &DL : D)
+ DF << DL << "\n";
+ }
+}
+
void DependencyFileGenerator::outputDependencyFile(DiagnosticsEngine &Diags) {
if (SeenMissingHeader) {
llvm::sys::fs::remove(OutputFile);
return;
}
+ std::string KDFN = GetKernelDepFileName(OutputFile);
std::error_code EC;
- llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_TextWithCRLF);
- if (EC) {
- Diags.Report(diag::err_fe_error_opening) << OutputFile << EC.message();
- return;
- }
+ // if need to merge kernel and host dep file
+ if (KDFN != "") {
+ // Read kernel dep file
+ std::vector<std::string> KD;
+ {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> KDF =
+ llvm::MemoryBuffer::getFile(KDFN);
+ if (KDF) {
+ llvm::StringRef KDC = KDF.get()->getBuffer();
+ KD = SplitToLines(KDC);
+ }
+ }
+
+ // open merged dep file
+ llvm::raw_fd_ostream DF(KDFN, EC, llvm::sys::fs::OF_Text);
+ if (EC) {
+ Diags.Report(diag::err_fe_error_opening) << KDFN << EC.message();
+ return;
+ }
+ // if KD is empty, just write host dep file to merged dep file
+ if (KD.empty())
+ outputDependencyFile(DF);
+ else {
+ // Get host dep file
+ std::vector<std::string> HD;
+ std::string HDC;
+ llvm::raw_string_ostream OSS(HDC);
+ outputDependencyFile(OSS);
+ llvm::StringRef HDCR(OSS.str());
+ if (!HDCR.empty()) {
+ HD = SplitToLines(HDCR);
+ // Merge kernel and host dep file
+ TryMergeDependencyFile(KD, HD, DF, Diags);
+ }
+ }
+ } else {
+ // merge is not needed, just write the dep file
+ llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::OF_Text);
+ if (EC) {
+ Diags.Report(diag::err_fe_error_opening) << OutputFile << EC.message();
+ return;
+ }
- outputDependencyFile(OS);
+ outputDependencyFile(OS);
+ }
}
void DependencyFileGenerator::outputDependencyFile(llvm::raw_ostream &OS) {
diff --git a/clang/test/ClangScanDeps/P1689.cppm b/clang/test/ClangScanDeps/P1689.cppm
index 4176a06ca3c34..d0cc98615a13c 100644
--- a/clang/test/ClangScanDeps/P1689.cppm
+++ b/clang/test/ClangScanDeps/P1689.cppm
@@ -35,10 +35,10 @@
// RUN: clang-scan-deps -format=p1689 \
// RUN: -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/impl_part.cppm -o %t/impl_part.o \
// RUN: -MT %t/impl_part.o.ddi -MD -MF %t/impl_part.dep
-// RUN: cat %t/impl_part.dep | FileCheck %t/impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE
+// RUN: cat %t/impl_part.dep* | FileCheck %t/impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE
//
// Check that we can generate multiple make-style dependency information with compilation database.
-// RUN: cat %t/P1689.dep | FileCheck %t/Checks.cpp -DPREFIX=%/t --check-prefix=CHECK-MAKE
+// RUN: cat %t/P1689.dep* | FileCheck %t/Checks.cpp -DPREFIX=%/t --check-prefix=CHECK-MAKE
//
// Check that we can mix the use of -format=p1689 and -fmodules.
// RUN: clang-scan-deps -format=p1689 \
@@ -50,7 +50,7 @@
// RUN: clang-scan-deps -format=p1689 \
// RUN: -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t impl_part.cppm -o impl_part.o \
// RUN: -MT impl_part.o.ddi -MD -MF impl_part.dep
-// RUN: cat impl_part.dep | FileCheck impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE-RELATIVE
+// RUN: cat impl_part.dep* | FileCheck impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE-RELATIVE
//--- P1689.json.in
diff --git a/clang/test/Driver/Wp-args.c b/clang/test/Driver/Wp-args.c
index 587b7b83e4ca6..15e37469cf1e7 100644
--- a/clang/test/Driver/Wp-args.c
+++ b/clang/test/Driver/Wp-args.c
@@ -7,7 +7,7 @@
//
// CHECK: "-cc1"
// CHECK-NOT: -MD
-// CHECK: "-dependency-file" "FOO.d"
+// CHECK: "-dependency-file" "FOO.d{{[^\"]*}}"
// CHECK: "-MT"
//
// PR4062
@@ -18,7 +18,7 @@
// MMD: "-cc1"
// MMD-NOT: -MMD
-// MMD: "-dependency-file" "Wp-args.d"
+// MMD: "-dependency-file" "Wp-args.d{{[^\"]*}}"
// Ensure response files are properly expanded with -Wp
// RUN: echo -DTEST > %t.rsp
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 9f9ca1bf1a8fd..79c69a7671166 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -759,7 +759,7 @@
// RUN: %clang_cl -O2 -MD /clang:-fno-slp-vectorize /clang:-MD /clang:-MF /clang:my_dependency_file.dep /c /Fo%/t/cl-options.obj -### -- %s 2>&1 | FileCheck -DPREFIX=%/t -check-prefix=CLANG %s
// CLANG: "--dependent-lib=msvcrt"
-// CLANG-SAME: "-dependency-file" "my_dependency_file.dep"
+// CLANG-SAME: "-dependency-file" "my_dependency_file.dep{{[^\"]*}}"
// CLANG-SAME: "-MT" "[[PREFIX]]/cl-options.obj"
// CLANG-NOT: "--dependent-lib=libcmt"
// CLANG-NOT: "-vectorize-slp"
diff --git a/clang/test/Driver/dep-file-flag-with-multiple-offload-archs.hip b/clang/test/Driver/dep-file-flag-with-multiple-offload-archs.hip
index f17e56acfb7f7..a115eac681b01 100644
--- a/clang/test/Driver/dep-file-flag-with-multiple-offload-archs.hip
+++ b/clang/test/Driver/dep-file-flag-with-multiple-offload-archs.hip
@@ -1,12 +1,12 @@
// RUN: %clang -### -nogpuinc -nogpulib --offload-arch=gfx1030 --offload-arch=gfx1100 --offload-arch=gfx1101 --target=x86_64-linux-gnu -MD -MF tmp.d %s 2>&1 | FileCheck %s
-// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1030"{{.*}}"-dependency-file" "tmp.d"
+// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1030"{{.*}}"-dependency-file" "tmp.d{{[^\"]*}}"
// CHECK: {{.*}}lld{{.*}}"-plugin-opt=mcpu=gfx1030"
-// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1100"{{.*}}"-dependency-file" "tmp.d"
+// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1100"{{.*}}"-dependency-file" "tmp.d{{[^\"]*}}"
// CHECK: {{.*}}lld{{.*}}"-plugin-opt=mcpu=gfx1100"
-// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1101"{{.*}}"-dependency-file" "tmp.d"
+// CHECK-NOT: {{.*}}clang{{.*}}"-target-cpu" "gfx1101"{{.*}}"-dependency-file" "tmp.d{{[^\"]*}}"
// CHECK: {{.*}}lld{{.*}}"-plugin-opt=mcpu=gfx1101"
// CHECK: {{.*}}clang-offload-bundler
-// CHECK: {{.*}}clang{{.*}}"-target-cpu"{{.*}}"-dependency-file" "tmp.d"
+// CHECK: {{.*}}clang{{.*}}"-target-cpu"{{.*}}"-dependency-file" "tmp.d{{[^\"]*}}"
void main(){}
diff --git a/clang/test/Driver/m-and-mm.c b/clang/test/Driver/m-and-mm.c
index cb719a6c97fe8..4c8486921d1c8 100644
--- a/clang/test/Driver/m-and-mm.c
+++ b/clang/test/Driver/m-and-mm.c
@@ -15,18 +15,18 @@
// RUN: %clang -MM %s -o %t.dir/test.i
// RUN: FileCheck %s < %t.dir/test.i
-// RUN: rm -f %t.dir/test.d
+// RUN: rm -f %t.dir/test.d*
// RUN: %clang -fsyntax-only -MD %s -o %t.dir/test.i
-// RUN: FileCheck --check-prefix=TEST-I %s < %t.dir/test.d
+// RUN: FileCheck --check-prefix=TEST-I %s < %t.dir/test.d*
-// RUN: rm -f %t.dir/test.d
+// RUN: rm -f %t.dir/test.d*
// RUN: %clang -M -MD %s -o %t.dir/test.i
-// RUN: FileCheck --check-prefix=TEST-I %s < %t.dir/test.d
+// RUN: FileCheck --check-prefix=TEST-I %s < %t.dir/test.d*
/// If the output file name does not have a suffix, just append `.d`.
-// RUN: rm -f %t.dir/test.d
+// RUN: rm -f %t.dir/test.d*
// RUN: %clang -fsyntax-only -MD %s -o %t.dir/test
-// RUN: FileCheck --check-prefix=TEST %s < %t.dir/test.d
+// RUN: FileCheck --check-prefix=TEST %s < %t.dir/test.d*
#warning "-M and -MM suppresses warnings, thus this warning shouldn't show up"
int main(void)
More information about the cfe-commits
mailing list