[clang] [HIP] Parallelize device cc1 jobs for offload arches (PR #208254)
Yaxun Liu via cfe-commits
cfe-commits at lists.llvm.org
Wed Jul 8 09:23:44 PDT 2026
https://github.com/yxsamliu created https://github.com/llvm/llvm-project/pull/208254
[HIP] Parallelize device cc1 jobs for offload arches
A common ROCm build can target around ten `--offload-arch=`
values. For a large single translation unit, the HIP device cc1 work
before clang-linker-wrapper can become a build-time bottleneck. Those
per-architecture jobs are independent, but the driver ran them serially,
so `--offload-jobs=` only helped the later wrapper work.
Borrow the parallel job mechanism used by clang-linker-wrapper for the
pre-wrapper HIP device cc1 jobs. HIP job construction marks eligible
compile and backend jobs with an offload parallel group. The generic
executor only consumes that opt-in metadata for adjacent jobs with
distinct bound architectures.
The parallel path is disabled for driver-side output modes and callbacks
so existing serial output handling is preserved. This is NFC for
non-offload compilation.
>From 4cc9abb105eb3f497ee80fcb430244854474a665 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 8 Jul 2026 12:13:26 -0400
Subject: [PATCH] [HIP] Parallelize device cc1 jobs for offload arches
A common ROCm build can target around ten `--offload-arch=`
values. For a large single translation unit, the HIP device cc1 work
before clang-linker-wrapper can become a build-time bottleneck. Those
per-architecture jobs are independent, but the driver ran them serially,
so `--offload-jobs=` only helped the later wrapper work.
Borrow the parallel job mechanism used by clang-linker-wrapper for the
pre-wrapper HIP device cc1 jobs. HIP job construction marks eligible
compile and backend jobs with an offload parallel group. The generic
executor only consumes that opt-in metadata for adjacent jobs with
distinct bound architectures.
The parallel path is disabled for driver-side output modes and callbacks
so existing serial output handling is preserved. This is NFC for
non-offload compilation.
---
clang/include/clang/Driver/Job.h | 14 +-
clang/lib/Driver/Compilation.cpp | 165 +++++++++++++++++-
clang/lib/Driver/ToolChains/Clang.cpp | 24 ++-
clang/test/Driver/hip-parallel-device-cc1.hip | 9 +
4 files changed, 206 insertions(+), 6 deletions(-)
create mode 100644 clang/test/Driver/hip-parallel-device-cc1.hip
diff --git a/clang/include/clang/Driver/Job.h b/clang/include/clang/Driver/Job.h
index 56a147e717237..03779be5b5a6a 100644
--- a/clang/include/clang/Driver/Job.h
+++ b/clang/include/clang/Driver/Job.h
@@ -151,10 +151,13 @@ class Command {
/// Information on executable run provided by OS.
mutable std::optional<llvm::sys::ProcessStatistics> ProcStat;
- /// The bound architecture for this command (e.g. "arm64", "x86_64").
- /// Non-empty only for Darwin multi-arch builds.
+ /// The bound architecture for this command (e.g. "arm64", "gfx90a").
std::string BoundArchStr;
+ /// Non-empty when this command may run in parallel with adjacent offload
+ /// device commands from the same group.
+ std::string OffloadDeviceParallelJobGroup;
+
/// When a response file is needed, we try to put most arguments in an
/// exclusive file, while others remains as regular command line arguments.
/// This functions fills a vector with the regular command line arguments,
@@ -199,6 +202,13 @@ class Command {
BoundArch getBoundArch() const { return BoundArch(BoundArchStr); }
void setBoundArch(BoundArch BA) { BoundArchStr = BA.ArchName.str(); }
+ StringRef getOffloadDeviceParallelJobGroup() const {
+ return OffloadDeviceParallelJobGroup;
+ }
+ void setOffloadDeviceParallelJobGroup(StringRef Group) {
+ OffloadDeviceParallelJobGroup = Group.str();
+ }
+
/// Returns the kind of response file supported by the current invocation.
const ResponseFileSupport &getResponseFileSupport() {
return ResponseSupport;
diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp
index 377ac7e2ad43e..f67d8c7f3e8b3 100644
--- a/clang/lib/Driver/Compilation.cpp
+++ b/clang/lib/Driver/Compilation.cpp
@@ -14,13 +14,18 @@
#include "clang/Driver/ToolChain.h"
#include "clang/Driver/Util.h"
#include "clang/Options/Options.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptSpecifier.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/Threading.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Triple.h"
+#include <algorithm>
#include <cassert>
+#include <optional>
#include <string>
#include <system_error>
#include <utility>
@@ -232,6 +237,139 @@ static bool ActionFailed(const Action *A,
return false;
}
+static bool ActionDependsOn(const Action *A, const Action *Other) {
+ if (A == Other)
+ return true;
+
+ for (const auto *Input : A->inputs())
+ if (ActionDependsOn(Input, Other))
+ return true;
+
+ return false;
+}
+
+static bool ActionsAreIndependent(const Action *A, const Action *B) {
+ return !ActionDependsOn(A, B) && !ActionDependsOn(B, A);
+}
+
+static bool CanRunInParallelOffloadJobGroup(const Command &Job) {
+ return !Job.InProcess && !Job.PrintInputFilenames &&
+ !Job.getBoundArch().empty() &&
+ !Job.getOffloadDeviceParallelJobGroup().empty();
+}
+
+static bool SameParallelOffloadJobGroup(const Command &A, const Command &B) {
+ return A.getOffloadDeviceParallelJobGroup() ==
+ B.getOffloadDeviceParallelJobGroup();
+}
+
+static bool HasDistinctBoundArch(const Command &Candidate,
+ ArrayRef<const Command *> Jobs) {
+ BoundArch CandidateArch = Candidate.getBoundArch();
+ return llvm::none_of(Jobs, [&](const Command *Job) {
+ return Job->getBoundArch() == CandidateArch;
+ });
+}
+
+static std::optional<llvm::ThreadPoolStrategy>
+getParallelOffloadJobsStrategy(const ArgList &Args, unsigned NumJobs) {
+ if (NumJobs < 2)
+ return std::nullopt;
+
+ Arg *A = Args.getLastArg(options::OPT_offload_jobs_EQ);
+ if (!A)
+ return std::nullopt;
+
+ StringRef Val = A->getValue();
+ if (Val.equals_insensitive("jobserver"))
+ return llvm::jobserver_concurrency();
+
+ unsigned NumThreads;
+ if (Val.getAsInteger(10, NumThreads) || NumThreads < 2)
+ return std::nullopt;
+
+ return llvm::hardware_concurrency(std::min(NumThreads, NumJobs));
+}
+
+struct ParallelJobResult {
+ int Res = 0;
+ bool ExecutionFailed = false;
+ std::string Error;
+};
+
+struct ParallelOffloadJobGroupResult {
+ size_t NumJobs = 0;
+ bool StopExecution = false;
+};
+
+static std::optional<ParallelOffloadJobGroupResult>
+tryExecuteParallelOffloadJobGroup(const Driver &D, const ArgList &Args,
+ ArrayRef<std::optional<StringRef>> Redirects,
+ const JobList::list_type &JobStorage,
+ size_t StartIndex,
+ FailingCommandList &FailingCommands) {
+ const Command &Job = *JobStorage[StartIndex];
+ if (!CanRunInParallelOffloadJobGroup(Job))
+ return std::nullopt;
+
+ SmallVector<const Command *, 4> ParallelJobs;
+ for (size_t I = StartIndex; I < JobStorage.size(); ++I) {
+ const Command &Candidate = *JobStorage[I];
+ if (!CanRunInParallelOffloadJobGroup(Candidate))
+ break;
+
+ if (!SameParallelOffloadJobGroup(Job, Candidate))
+ break;
+
+ if (!HasDistinctBoundArch(Candidate, ParallelJobs))
+ break;
+
+ if (!llvm::all_of(ParallelJobs, [&](const Command *Other) {
+ return ActionsAreIndependent(&Candidate.getSource(),
+ &Other->getSource());
+ }))
+ break;
+
+ ParallelJobs.push_back(&Candidate);
+ }
+
+ std::optional<llvm::ThreadPoolStrategy> Strategy =
+ getParallelOffloadJobsStrategy(Args, ParallelJobs.size());
+ if (!Strategy)
+ return std::nullopt;
+
+ SmallVector<ParallelJobResult, 4> Results(ParallelJobs.size());
+ llvm::DefaultThreadPool Pool(*Strategy);
+ for (auto [Index, ParallelJob] : llvm::enumerate(ParallelJobs)) {
+ Pool.async([&, Index, ParallelJob] {
+ Results[Index].Res = ParallelJob->Execute(
+ Redirects, &Results[Index].Error, &Results[Index].ExecutionFailed);
+ });
+ }
+ Pool.wait();
+
+ bool StopExecution = false;
+ for (auto [Index, ParallelJob] : llvm::enumerate(ParallelJobs)) {
+ ParallelJobResult &Result = Results[Index];
+ if (!Result.Error.empty()) {
+ assert(Result.Res && "Error string set with 0 result code!");
+ D.Diag(diag::err_drv_command_failure) << Result.Error;
+ }
+
+ if (Result.Res) {
+ FailingCommands.push_back(
+ std::make_pair(Result.ExecutionFailed ? 1 : Result.Res, ParallelJob));
+ // Bail as soon as one command fails in cl driver mode.
+ if (D.IsCLMode()) {
+ StopExecution = true;
+ break;
+ }
+ }
+ }
+
+ return ParallelOffloadJobGroupResult{ParallelJobs.size(), StopExecution};
+}
+
void Compilation::ExecuteJobs(const JobList &Jobs,
FailingCommandList &FailingCommands,
bool LogOnly) const {
@@ -239,9 +377,31 @@ void Compilation::ExecuteJobs(const JobList &Jobs,
// inputs on the command line even one of them failed.
// In all but CLMode, execute all the jobs unless the necessary inputs for the
// job is missing due to previous failures.
- for (const auto &Job : Jobs) {
- if (ActionFailed(&Job.getSource(), FailingCommands))
+ bool CanRunJobsInParallel =
+ !LogOnly && !getDriver().CCPrintOptions &&
+ !getDriver().CCPrintProcessStats && !getDriver().CCGenDiagnostics &&
+ !getArgs().hasArg(options::OPT_v) && Redirects.empty() && !PostCallback;
+
+ const auto &JobStorage = Jobs.getJobs();
+ for (size_t I = 0; I < JobStorage.size();) {
+ const auto &Job = *JobStorage[I];
+ if (ActionFailed(&Job.getSource(), FailingCommands)) {
+ ++I;
continue;
+ }
+
+ if (CanRunJobsInParallel) {
+ if (std::optional<ParallelOffloadJobGroupResult> Result =
+ tryExecuteParallelOffloadJobGroup(getDriver(), getArgs(),
+ Redirects, JobStorage, I,
+ FailingCommands)) {
+ if (Result->StopExecution)
+ return;
+ I += Result->NumJobs;
+ continue;
+ }
+ }
+
const Command *FailingCommand = nullptr;
if (int Res = ExecuteCommand(Job, FailingCommand, LogOnly)) {
FailingCommands.push_back(std::make_pair(Res, FailingCommand));
@@ -249,6 +409,7 @@ void Compilation::ExecuteJobs(const JobList &Jobs,
if (TheDriver.IsCLMode())
return;
}
+ ++I;
}
}
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index a30b01a675b99..d9bb7622ea1da 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5132,6 +5132,22 @@ static void ProcessVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args,
CmdArgs.push_back("--dependent-lib=softintrin");
}
+static bool CanRunOffloadDeviceCC1JobInParallel(const JobAction &JA) {
+ if (JA.getOffloadingArch().empty())
+ return false;
+
+ if (!JA.isDeviceOffloading(Action::OFK_HIP))
+ return false;
+
+ return isa<CompileJobAction>(JA) || isa<BackendJobAction>(JA);
+}
+
+static std::string getOffloadDeviceCC1ParallelJobGroup(const JobAction &JA) {
+ return (Twine(Action::GetOffloadKindName(JA.getOffloadingDeviceKind())) +
+ ":" + JA.getClassName() + ":" + types::getTypeName(JA.getType()))
+ .str();
+}
+
void Clang::ConstructJob(Compilation &C, const JobAction &JA,
const InputInfo &Output, const InputInfoList &Inputs,
const ArgList &Args, const char *LinkingOutput) const {
@@ -8557,9 +8573,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs,
Output, D.getPrependArg()));
} else {
- C.addCommand(std::make_unique<Command>(
+ auto Cmd = std::make_unique<Command>(
JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs,
- Output, D.getPrependArg()));
+ Output, D.getPrependArg());
+ if (CanRunOffloadDeviceCC1JobInParallel(JA))
+ Cmd->setOffloadDeviceParallelJobGroup(
+ getOffloadDeviceCC1ParallelJobGroup(JA));
+ C.addCommand(std::move(Cmd));
}
// Make the compile command echo its inputs for /showFilenames.
diff --git a/clang/test/Driver/hip-parallel-device-cc1.hip b/clang/test/Driver/hip-parallel-device-cc1.hip
new file mode 100644
index 0000000000000..97b754efbf2aa
--- /dev/null
+++ b/clang/test/Driver/hip-parallel-device-cc1.hip
@@ -0,0 +1,9 @@
+// REQUIRES: x86-registered-target, amdgpu-registered-target, lld
+
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: -nostdinc -nogpuinc -nohipwrapperinc -nogpulib \
+// RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \
+// RUN: -O3 -c %s -o %t.o
+
+// Empty source file. The RUN line is an execution smoke test for the driver
+// path that runs independent HIP device cc1 jobs through --offload-jobs.
More information about the cfe-commits
mailing list