[clang] [llvm] [Offload] Split frontend CUDA/HIP language assumptions from backend device (PR #210121)
Sophia Herrmann via cfe-commits
cfe-commits at lists.llvm.org
Sun Jul 19 19:20:31 PDT 2026
https://github.com/jellytabby updated https://github.com/llvm/llvm-project/pull/210121
>From b7f3cd416e0d6a11867567398fb739d877f702d9 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes at jdoerfert.de>
Date: Thu, 18 Jun 2026 14:13:36 -0700
Subject: [PATCH 01/23] [Offload] WIP offload via llvm
---
clang/lib/CodeGen/CGCUDANV.cpp | 32 +-
clang/lib/Driver/Driver.cpp | 19 +-
clang/lib/Driver/ToolChains/Clang.cpp | 7 +-
clang/lib/Driver/ToolChains/CommonArgs.cpp | 14 +-
clang/lib/Driver/ToolChains/Cuda.cpp | 23 +-
.../llvm_offload_wrappers/__llvm_offload.h | 54 +++
.../__llvm_offload_host.h | 4 +-
.../Offloading/Utility_BACKUP_865488.h | 258 +++++++++++++++
.../Frontend/Offloading/Utility_BASE_865488.h | 231 +++++++++++++
.../Offloading/Utility_LOCAL_865488.h | 240 ++++++++++++++
.../Offloading/Utility_REMOTE_865488.h | 247 ++++++++++++++
llvm/include/llvm/Object/OffloadBinary.h | 3 +-
.../Frontend/Offloading/OffloadWrapper.cpp | 4 +-
llvm/llvm.code-workspace | 23 ++
...gate-waves-per-eu-no-kernel.ll.kernels.inc | 15 +
.../propagate-waves-per-eu-with-kernel.ll | 81 +++++
offload/CMakeLists.txt | 1 +
offload/languages/CMakeLists.txt | 3 +
offload/languages/cuda/CMakeLists.txt | 42 +++
offload/languages/cuda/exports | 10 +
offload/languages/cuda/src/cuda_runtime.cpp | 27 ++
offload/languages/hip/CMakeLists.txt | 42 +++
offload/languages/hip/exports | 10 +
offload/languages/hip/src/hip_runtime.cpp | 31 ++
offload/languages/include/cuda/cuda_runtime.h | 22 ++
offload/languages/include/hip/hip_runtime.h | 47 +++
.../include/kernel/DefineLanguageNames.inc | 37 +++
.../include/kernel/LanguageRuntime.h | 165 ++++++++++
.../include/kernel/UndefineLanguageNames.inc | 34 ++
offload/languages/kernel/CMakeLists.txt | 41 +++
offload/languages/kernel/exports | 8 +
.../languages/kernel/include/ExportedAPI.h | 31 ++
.../kernel/include/LanguageAliases.h | 40 +++
.../languages/kernel/include/LanguageLaunch.h | 51 +++
.../kernel/include/LanguageRegistration.h | 62 ++++
.../languages/kernel/include/Registration.h | 17 +
offload/languages/kernel/include/State.h | 92 ++++++
offload/languages/kernel/include/Types.h | 28 ++
offload/languages/kernel/src/ExportedAPI.cpp | 60 ++++
.../languages/kernel/src/LanguageLaunch.cpp | 94 ++++++
.../kernel/src/LanguageRegistration.cpp | 311 ++++++++++++++++++
.../languages/kernel/src/LanguageRuntime.cpp | 152 +++++++++
offload/languages/kernel/src/State.cpp | 167 ++++++++++
offload/liboffload/API/Device.td | 24 ++
offload/liboffload/src/OffloadImpl.cpp | 19 ++
offload/libomptarget/CMakeLists.txt | 2 -
offload/libomptarget/KernelLanguage/API.cpp | 74 -----
offload/libomptarget/exports | 3 -
offload/test/api/kernel_handle.cpp | 38 +++
offload/test/lit.cfg | 8 +
offload/test/offloading/CUDA/basic_launch.cu | 22 +-
.../CUDA/basic_launch_blocks_and_threads.cu | 19 +-
.../offloading/CUDA/basic_launch_multi_arg.cu | 33 +-
offload/test/offloading/CUDA/launch_tu.cu | 20 +-
.../offloading/CUDA/thread_and_block_id.cu | 41 +++
.../OffloadAPI/kernel/olGetKernel.cpp | 59 ++++
.../kernel/olGetKernelProperties.cpp | 62 ++++
57 files changed, 3134 insertions(+), 170 deletions(-)
create mode 100644 llvm/include/llvm/Frontend/Offloading/Utility_BACKUP_865488.h
create mode 100644 llvm/include/llvm/Frontend/Offloading/Utility_BASE_865488.h
create mode 100644 llvm/include/llvm/Frontend/Offloading/Utility_LOCAL_865488.h
create mode 100644 llvm/include/llvm/Frontend/Offloading/Utility_REMOTE_865488.h
create mode 100644 llvm/llvm.code-workspace
create mode 100644 llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-no-kernel.ll.kernels.inc
create mode 100644 llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-with-kernel.ll
create mode 100644 offload/languages/CMakeLists.txt
create mode 100644 offload/languages/cuda/CMakeLists.txt
create mode 100644 offload/languages/cuda/exports
create mode 100644 offload/languages/cuda/src/cuda_runtime.cpp
create mode 100644 offload/languages/hip/CMakeLists.txt
create mode 100644 offload/languages/hip/exports
create mode 100644 offload/languages/hip/src/hip_runtime.cpp
create mode 100644 offload/languages/include/cuda/cuda_runtime.h
create mode 100644 offload/languages/include/hip/hip_runtime.h
create mode 100644 offload/languages/include/kernel/DefineLanguageNames.inc
create mode 100644 offload/languages/include/kernel/LanguageRuntime.h
create mode 100644 offload/languages/include/kernel/UndefineLanguageNames.inc
create mode 100644 offload/languages/kernel/CMakeLists.txt
create mode 100644 offload/languages/kernel/exports
create mode 100644 offload/languages/kernel/include/ExportedAPI.h
create mode 100644 offload/languages/kernel/include/LanguageAliases.h
create mode 100644 offload/languages/kernel/include/LanguageLaunch.h
create mode 100644 offload/languages/kernel/include/LanguageRegistration.h
create mode 100644 offload/languages/kernel/include/Registration.h
create mode 100644 offload/languages/kernel/include/State.h
create mode 100644 offload/languages/kernel/include/Types.h
create mode 100644 offload/languages/kernel/src/ExportedAPI.cpp
create mode 100644 offload/languages/kernel/src/LanguageLaunch.cpp
create mode 100644 offload/languages/kernel/src/LanguageRegistration.cpp
create mode 100644 offload/languages/kernel/src/LanguageRuntime.cpp
create mode 100644 offload/languages/kernel/src/State.cpp
delete mode 100644 offload/libomptarget/KernelLanguage/API.cpp
create mode 100644 offload/test/api/kernel_handle.cpp
create mode 100644 offload/test/offloading/CUDA/thread_and_block_id.cu
create mode 100644 offload/unittests/OffloadAPI/kernel/olGetKernel.cpp
create mode 100644 offload/unittests/OffloadAPI/kernel/olGetKernelProperties.cpp
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 11bdc457cf9ff..5b46d5c33dfe0 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -252,9 +252,7 @@ CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
VoidTy = CGM.VoidTy;
PtrTy = CGM.DefaultPtrTy;
- if (CGM.getLangOpts().OffloadViaLLVM)
- Prefix = "llvm";
- else if (CGM.getLangOpts().HIP)
+ if (CGM.getLangOpts().HIP)
Prefix = "hip";
else
Prefix = "cuda";
@@ -366,6 +364,12 @@ Address CGNVCUDARuntime::prepareKernelArgsLLVMOffload(CodeGenFunction &CGF,
Address KernelLaunchParams = CGF.CreateTempAllocaWithoutCast(
KernelLaunchParamsTy, CharUnits::fromQuantity(16),
"kernel_launch_params");
+ Address KernelArgs = CGF.CreateTempAlloca(
+ PtrTy, LangAS::Default, CharUnits::fromQuantity(16), "kernel_args",
+ llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size())));
+ Address KernelArgSizes = CGF.CreateTempAlloca(
+ SizeTy, LangAS::Default, CharUnits::fromQuantity(16), "kernel_arg_sizes",
+ llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size())));
CGF.Builder.CreateStore(llvm::ConstantInt::get(Int32Ty, Args.size()),
CGF.Builder.CreateStructGEP(KernelLaunchParams, 0));
@@ -406,10 +410,11 @@ Address CGNVCUDARuntime::prepareKernelArgs(CodeGenFunction &CGF,
// array and kernels are launched using cudaLaunchKernel().
void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
FunctionArgList &Args) {
+ bool UseLLVMOffload = CGF.getLangOpts().OffloadViaLLVM;
// Build the shadow stack entry at the very start of the function.
- Address KernelArgs = CGF.getLangOpts().OffloadViaLLVM
- ? prepareKernelArgsLLVMOffload(CGF, Args)
- : prepareKernelArgs(CGF, Args);
+ Address KernelArgs = UseLLVMOffload
+ ? prepareKernelArgsLLVMOffload(CGF, Args) :
+ prepareKernelArgs(CGF, Args);
llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
@@ -433,7 +438,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
else if (CGF.getLangOpts().CUDA)
KernelLaunchAPI = KernelLaunchAPI + "_ptsz";
}
- auto LaunchKernelName = addPrefixToName(KernelLaunchAPI);
+ /// Use __llvmLaunchKernel for LLVMOffload.
+ auto LaunchKernelName = UseLLVMOffload ? "__llvm" + KernelLaunchAPI : addPrefixToName(KernelLaunchAPI);
const IdentifierInfo &cudaLaunchKernelII =
CGM.getContext().Idents.get(LaunchKernelName);
FunctionDecl *cudaLaunchKernelFD = nullptr;
@@ -951,7 +957,10 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
// Data.
Values.add(FatBinStr);
// Unused in fatbin v1.
- Values.add(llvm::ConstantPointerNull::get(PtrTy));
+ if (CGM.getLangOpts().OffloadViaLLVM && CudaGpuBinary)
+ Values.add(llvm::ConstantExpr::getGetElementPtr(CGM.Int8Ty, FatBinStr, llvm::ConstantInt::get(CGM.Int32Ty, CudaGpuBinary->getBuffer().size())));
+ else
+ Values.add(llvm::ConstantPointerNull::get(PtrTy));
llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal(
addUnderscoredPrefixToName("_fatbin_wrapper"), CGM.getPointerAlign(),
/*constant*/ true);
@@ -1270,10 +1279,13 @@ void CGNVCUDARuntime::createOffloadingEntries() {
llvm::object::OffloadKind Kind = CGM.getLangOpts().HIP
? llvm::object::OffloadKind::OFK_HIP
: llvm::object::OffloadKind::OFK_Cuda;
- // For now, just spoof this as OpenMP because that's the runtime it uses.
+
+ // For offload via llvm it doesn't matter if the source is HIP or CUDA or
+ // something else. The bundler will allow LLVM offload kinds for all languages.
if (CGM.getLangOpts().OffloadViaLLVM)
- Kind = llvm::object::OffloadKind::OFK_OpenMP;
+ Kind = llvm::object::OffloadKind::OFK_LLVM;
+ llvm::errs() << __PRETTY_FUNCTION__ << " : : " << EmittedKernels.size() << "\n";
llvm::Module &M = CGM.getModule();
for (KernelInfo &I : EmittedKernels)
llvm::offloading::emitOffloadingEntry(
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 1a91b353a27e4..7144069ec0207 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -1016,20 +1016,18 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
InputList &Inputs) {
bool UseLLVMOffload = C.getInputArgs().hasArg(
options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
- bool IsCuda =
+ bool IsCuda = !UseLLVMOffload &&
llvm::any_of(Inputs,
[](std::pair<types::ID, const llvm::opt::Arg *> &I) {
return types::isCuda(I.first);
- }) &&
- !UseLLVMOffload;
- bool IsHIP =
+ });
+ bool IsHIP = !UseLLVMOffload &&
(llvm::any_of(Inputs,
[](std::pair<types::ID, const llvm::opt::Arg *> &I) {
return types::isHIP(I.first);
}) ||
C.getInputArgs().hasArg(options::OPT_hip_link) ||
- C.getInputArgs().hasArg(options::OPT_hipstdpar)) &&
- !UseLLVMOffload;
+ C.getInputArgs().hasArg(options::OPT_hipstdpar));
bool IsSYCL = C.getInputArgs().hasFlag(options::OPT_fsycl,
options::OPT_fno_sycl, false);
bool IsOpenMPOffloading =
@@ -5038,6 +5036,9 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
getFinalPhase(Args) == phases::Preprocess))
return HostAction;
+ bool UseLLVMOffload = Args.hasArg(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
+
ActionList OffloadActions;
OffloadAction::DeviceDependences DDeps;
@@ -5058,9 +5059,9 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
types::ID InputType = Input.first;
const Arg *InputArg = Input.second;
- // The toolchain can be active for unsupported file types.
- if ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
- (Kind == Action::OFK_HIP && !types::isHIP(InputType)))
+ // Allow the toolchain to be active for unsupported file types if we are "offload-cross-compiling" via llvm-offload.
+ if (!UseLLVMOffload && ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
+ (Kind == Action::OFK_HIP && !types::isHIP(InputType))))
continue;
// Get the product of all bound architectures and toolchains.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index a30b01a675b99..d1b03d12ae07f 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -986,10 +986,13 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
SmallString<128> P(D.ResourceDir);
llvm::sys::path::append(P, "include", "llvm_offload_wrappers");
CmdArgs.append({"-internal-isystem", Args.MakeArgString(P), "-include"});
- if (JA.isDeviceOffloading(Action::OFK_OpenMP))
+ if (JA.isDeviceOffloading(Action::OFK_OpenMP)) {
CmdArgs.push_back("__llvm_offload_device.h");
- else
+ } else {
CmdArgs.push_back("__llvm_offload_host.h");
+ }
+ CmdArgs.push_back("-include");
+ CmdArgs.push_back("cuda_runtime.h");
}
// Add -i* options, and automatically translate to
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index dfd13fbe8a4eb..e89dd00cebd04 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1458,14 +1458,16 @@ bool tools::addOpenMPRuntime(const Compilation &C, ArgStringList &CmdArgs,
const ToolChain &TC, const ArgList &Args,
bool ForceStaticHostRuntime, bool IsOffloadingHost,
bool GompNeedsRT) {
+ bool IsOffloadViaLLVM = Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+ if (IsOffloadViaLLVM && IsOffloadingHost) {
+ CmdArgs.push_back("-lLLVMhip64");
+ CmdArgs.push_back("-lLLVMcudart");
+ }
+
if (!Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
- options::OPT_fno_openmp, false)) {
- // We need libomptarget (liboffload) if it's the choosen offloading runtime.
- if (Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false))
- CmdArgs.push_back("-lomptarget");
+ options::OPT_fno_openmp, false))
return false;
- }
Driver::OpenMPRuntimeKind RTKind = TC.getDriver().getOpenMPRuntime(Args);
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 9590ff976275c..e8d9c59f05782 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -301,6 +301,10 @@ CudaInstallationDetector::CudaInstallationDetector(
void CudaInstallationDetector::AddCudaIncludeArgs(
const ArgList &DriverArgs, ArgStringList &CC1Args) const {
+ if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false))
+ return;
+
if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
// Add cuda_wrappers/* to our system include path. This lets us wrap
// standard library headers.
@@ -395,7 +399,10 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
const char *LinkingOutput) const {
const auto &TC =
static_cast<const toolchains::NVPTXToolChain &>(getToolChain());
- assert(TC.getTriple().isNVPTX() && "Wrong platform");
+
+ bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+ assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
BoundArch GPUArch;
// If this is a CUDA action we need to extract the device architecture
@@ -418,7 +425,7 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
"Device action expected to have an architecture.");
// Check that our installation's ptxas supports gpu_arch.
- if (!Args.hasArg(options::OPT_no_cuda_version_check)) {
+ if (!UseLLVMOffload && !Args.hasArg(options::OPT_no_cuda_version_check)) {
TC.CudaInstallation.CheckCudaVersionSupportsArch(GPUArch.Arch);
}
@@ -540,7 +547,9 @@ void NVPTX::FatBinary::ConstructJob(Compilation &C, const JobAction &JA,
const char *LinkingOutput) const {
const auto &TC =
static_cast<const toolchains::CudaToolChain &>(getToolChain());
- assert(TC.getTriple().isNVPTX() && "Wrong platform");
+ bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+ assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
ArgStringList CmdArgs;
if (TC.CudaInstallation.version() <= CudaVersion::CUDA_100)
@@ -588,7 +597,9 @@ void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
static_cast<const toolchains::NVPTXToolChain &>(getToolChain());
ArgStringList CmdArgs;
- assert(TC.getTriple().isNVPTX() && "Wrong platform");
+ bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+ assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
assert((Output.isFilename() || Output.isNothing()) && "Invalid output.");
if (Output.isFilename()) {
@@ -963,6 +974,10 @@ llvm::DenormalMode CudaToolChain::getDefaultDenormalModeForType(
void CudaToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,
ArgStringList &CC1Args) const {
+ if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false))
+ return;
+
// Check our CUDA version if we're going to include the CUDA headers.
if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
true) &&
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
index 2898898904e29..7f218e235956c 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
@@ -9,6 +9,11 @@
#include <stddef.h>
+#pragma push_macro("_OPENMP")
+#define _OPENMP
+#include <gpuintrin.h>
+#pragma pop_macro("_OPENMP")
+
#define __host__ __attribute__((host))
#define __device__ __attribute__((device))
#define __global__ __attribute__((global))
@@ -29,3 +34,52 @@ typedef struct dim3 {
unsigned __llvmPushCallConfiguration(dim3 gridDim, dim3 blockDim,
size_t sharedMem = 0, void *stream = 0);
}
+
+// Make sure nobody can create instances of the coordinate types, take their
+// address, copy, or assign them.
+#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+#define __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag) \
+ __attribute__((device)) __tag() = delete; \
+ __attribute__((device)) __tag(const __tag &) = delete; \
+ __attribute__((device)) void operator=(const __tag &) const = delete; \
+ __attribute__((device)) __tag *operator&() const = delete
+
+#pragma push_macro("__GPU_COORD_BUILTIN")
+#define __GPU_COORD_BUILTIN(__tag, __fx, __fy, __fz) \
+ struct __tag { \
+ __declspec(property(get = __get_x)) unsigned int x; \
+ __declspec(property(get = __get_y)) unsigned int y; \
+ __declspec(property(get = __get_z)) unsigned int z; \
+ static inline __attribute__((device, always_inline)) unsigned int \
+ __get_x() { \
+ return __fx; \
+ } \
+ static inline __attribute__((device, always_inline)) unsigned int \
+ __get_y() { \
+ return __fy; \
+ } \
+ static inline __attribute__((device, always_inline)) unsigned int \
+ __get_z() { \
+ return __fz; \
+ } \
+ \
+ private: \
+ __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag); \
+ }
+
+__GPU_COORD_BUILTIN(__gpu_builtin_threadIdx_t, __gpu_thread_id_x(),
+ __gpu_thread_id_y(), __gpu_thread_id_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_blockIdx_t, __gpu_block_id_x(),
+ __gpu_block_id_y(), __gpu_block_id_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_blockDim_t, __gpu_num_threads_x(),
+ __gpu_num_threads_y(), __gpu_num_threads_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_gridDim_t, __gpu_num_blocks_x(),
+ __gpu_num_blocks_y(), __gpu_num_blocks_z());
+
+#pragma pop_macro("__GPU_COORD_BUILTIN")
+#pragma pop_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+
+extern const __attribute__((device, weak)) __gpu_builtin_threadIdx_t threadIdx;
+extern const __attribute__((device, weak)) __gpu_builtin_blockIdx_t blockIdx;
+extern const __attribute__((device, weak)) __gpu_builtin_blockDim_t blockDim;
+extern const __attribute__((device, weak)) __gpu_builtin_gridDim_t gridDim;
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_host.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_host.h
index 160289d169b55..b05cd852bc0f4 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_host.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_host.h
@@ -10,6 +10,6 @@
#include "__llvm_offload.h"
extern "C" {
-unsigned llvmLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
- void **args, size_t sharedMem = 0, void *stream = 0);
+unsigned __llvmLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
+ void **args, size_t sharedMem = 0, void *stream = 0);
}
diff --git a/llvm/include/llvm/Frontend/Offloading/Utility_BACKUP_865488.h b/llvm/include/llvm/Frontend/Offloading/Utility_BACKUP_865488.h
new file mode 100644
index 0000000000000..7ccc7b521e573
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Offloading/Utility_BACKUP_865488.h
@@ -0,0 +1,258 @@
+//===- Utility.h - Collection of geneirc offloading utilities -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OFFLOADING_UTILITY_H
+#define LLVM_FRONTEND_OFFLOADING_UTILITY_H
+
+#include "llvm/Support/Compiler.h"
+#include <cstdint>
+#include <memory>
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/OffloadBinary.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBufferRef.h"
+
+namespace llvm {
+namespace offloading {
+
+/// This is the record of an object that just be registered with the offloading
+/// runtime.
+struct EntryTy {
+ /// Reserved bytes used to detect an older version of the struct, always zero.
+ uint64_t Reserved = 0x0;
+ /// The current version of the struct for runtime forward compatibility.
+ uint16_t Version = 0x1;
+ /// The expected consumer of this entry, e.g. CUDA or OpenMP.
+ uint16_t Kind;
+ /// Flags associated with the global.
+ uint32_t Flags;
+ /// The address of the global to be registered by the runtime.
+ void *Address;
+ /// The name of the symbol in the device image.
+ char *SymbolName;
+ /// The number of bytes the symbol takes.
+ uint64_t Size;
+ /// Extra generic data used to register this entry.
+ uint64_t Data;
+ /// An extra pointer, usually null.
+ void *AuxAddr;
+};
+
+/// Offloading entry flags for CUDA / HIP. The first three bits indicate the
+/// type of entry while the others are a bit field for additional information.
+enum OffloadEntryKindFlag : uint32_t {
+ /// Mark the entry as a global entry. This indicates the presense of a
+ /// kernel if the size size field is zero and a variable otherwise.
+ OffloadGlobalEntry = 0x0,
+ /// Mark the entry as a managed global variable.
+ OffloadGlobalManagedEntry = 0x1,
+ /// Mark the entry as a surface variable.
+ OffloadGlobalSurfaceEntry = 0x2,
+ /// Mark the entry as a texture variable.
+ OffloadGlobalTextureEntry = 0x3,
+ /// Mark the entry as being extern.
+ OffloadGlobalExtern = 0x1 << 3,
+ /// Mark the entry as being constant.
+ OffloadGlobalConstant = 0x1 << 4,
+ /// Mark the entry as being a normalized surface.
+ OffloadGlobalNormalized = 0x1 << 5,
+};
+
+/// Returns the type of the offloading entry we use to store kernels and
+/// globals that will be registered with the offloading runtime.
+LLVM_ABI StructType *getEntryTy(Module &M);
+
+/// Create an offloading section struct used to register this global at
+/// runtime.
+///
+/// \param M The module to be used
+/// \param Addr The pointer to the global being registered.
+/// \param Kind The offloading language expected to consume this.
+/// \param Name The symbol name associated with the global.
+/// \param Size The size in bytes of the global (0 for functions).
+/// \param Flags Flags associated with the entry.
+/// \param Data Extra data storage associated with the entry.
+/// \param SectionName The section this entry will be placed at.
+/// \param AuxAddr An extra pointer if needed.
+/// Returns the section name for offloading entries based on the target triple.
+/// ELF: "llvm_offload_entries", COFF: "llvm_offload_entries",
+/// Mach-O: "__LLVM,offload_entries".
+LLVM_ABI StringRef getOffloadEntrySection(Module &M);
+
+/// \return The emitted global variable containing the offloading entry.
+LLVM_ABI GlobalVariable *
+emitOffloadingEntry(Module &M, object::OffloadKind Kind, Constant *Addr,
+ StringRef Name, uint64_t Size, uint32_t Flags,
+ uint64_t Data, Constant *AuxAddr = nullptr);
+
+/// Create a constant struct initializer used to register this global at
+/// runtime.
+/// \return the constant struct and the global variable holding the symbol name.
+LLVM_ABI std::pair<Constant *, GlobalVariable *>
+getOffloadingEntryInitializer(Module &M, object::OffloadKind Kind,
+ Constant *Addr, StringRef Name, uint64_t Size,
+ uint32_t Flags, uint64_t Data, Constant *AuxAddr);
+
+/// Creates a pair of globals used to iterate the array of offloading entries by
+/// accessing the section variables provided by the linker.
+LLVM_ABI std::pair<GlobalVariable *, GlobalVariable *>
+getOffloadEntryArray(Module &M);
+
+namespace amdgpu {
+/// Check if an image is compatible with current system's environment. The
+/// system environment is given as a 'target-id' which has the form:
+///
+/// <target-id> := <processor> ( ":" <target-feature> ( "+" | "-" ) )*
+///
+/// If a feature is not specific as '+' or '-' it is assumed to be in an 'any'
+/// and is compatible with either '+' or '-'. The HSA runtime returns this
+/// information using the target-id, while we use the ELF header to determine
+/// these features.
+LLVM_ABI bool isImageCompatibleWithEnv(StringRef ImageArch, uint32_t ImageFlags,
+ StringRef EnvTargetID);
+
+/// The offset and size of an AMD GPU kernel argument, as read from MetaData.
+struct AMDGPUKernelArgumentInfo {
+ // Address space and value kind are available but not stored right now.
+ uint32_t Offset;
+ uint32_t Size;
+};
+
+/// Container for the kernel argument layout, as read from MetaData.
+struct AMDGPUKernelArgumentLayout {
+ SmallVector<AMDGPUKernelArgumentInfo, 32> Arguments;
+ int32_t NumUserArguments = -1;
+};
+
+/// Struct for holding metadata related to AMDGPU kernels, for more information
+/// about the metadata and its meaning see:
+/// https://llvm.org/docs/AMDGPUUsage.html#code-object-v3
+struct AMDGPUKernelMetaData {
+ /// Constant indicating that a value is invalid.
+ static constexpr uint32_t KInvalidValue =
+ std::numeric_limits<uint32_t>::max();
+ /// The amount of group segment memory required by a work-group in bytes.
+ uint32_t GroupSegmentList = KInvalidValue;
+ /// The amount of fixed private address space memory required for a work-item
+ /// in bytes.
+ uint32_t PrivateSegmentSize = KInvalidValue;
+ /// Number of scalar registers required by a wavefront.
+ uint32_t SGPRCount = KInvalidValue;
+ /// Number of vector registers required by each work-item.
+ uint32_t VGPRCount = KInvalidValue;
+ /// Number of stores from a scalar register to a register allocator created
+ /// spill location.
+ uint32_t SGPRSpillCount = KInvalidValue;
+ /// Number of stores from a vector register to a register allocator created
+ /// spill location.
+ uint32_t VGPRSpillCount = KInvalidValue;
+ /// Number of accumulator registers required by each work-item.
+ uint32_t AGPRCount = KInvalidValue;
+ /// Corresponds to the OpenCL reqd_work_group_size attribute.
+ uint32_t RequestedWorkgroupSize[3] = {KInvalidValue, KInvalidValue,
+ KInvalidValue};
+ /// Corresponds to the OpenCL work_group_size_hint attribute.
+ uint32_t WorkgroupSizeHint[3] = {KInvalidValue, KInvalidValue, KInvalidValue};
+ /// Wavefront size.
+ uint32_t WavefrontSize = KInvalidValue;
+ /// Maximum flat work-group size supported by the kernel in work-items.
+ uint32_t MaxFlatWorkgroupSize = KInvalidValue;
+<<<<<<< HEAD
+ /// Per-argument {offset, size} in bytes, read from the ".args" array in code
+ /// object metadata. Explicit user arguments are first, followed by
+ /// hidden arguments.
+ SmallVector<std::pair<uint32_t, uint32_t>, 8> ArgMDs;
+=======
+ /// The argument layout of this kernel.
+ AMDGPUKernelArgumentLayout ArgumentLayout;
+>>>>>>> 07450dabab1c ([WIP] CUDA/HIP via llvm offload)
+};
+
+/// Reads AMDGPU specific metadata from the ELF file and propagates the
+/// KernelInfoMap.
+LLVM_ABI Error getAMDGPUMetaDataFromImage(
+ MemoryBufferRef MemBuffer, StringMap<AMDGPUKernelMetaData> &KernelInfoMap,
+ uint16_t &ELFABIVersion);
+} // namespace amdgpu
+
+/// Containerizes an image within an OffloadBinary image.
+/// Creates a nested OffloadBinary structure where the inner binary contains
+/// the raw image and associated metadata (version, format, triple, etc.).
+/// \param Binary The image to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param ImageKind The format of the image, e.g. SPIR-V or CUBIN.
+/// \param OffloadKind The expected consuming runtime of the image, e.g. CUDA or
+/// OpenMP.
+/// \param ImageFlags Flags associated with the image, e.g. for AMDGPU the
+/// features.
+/// \param MetaData The key-value map of metadata to be associated with the
+/// image.
+LLVM_ABI Error containerizeImage(std::unique_ptr<MemoryBuffer> &Binary,
+ llvm::Triple Triple,
+ object::ImageKind ImageKind,
+ object::OffloadKind OffloadKind,
+ int32_t ImageFlags,
+ MapVector<StringRef, StringRef> &MetaData);
+
+namespace sycl {
+
+/// Serialized symbol table stored in the "symbols" entry of a SYCL
+/// OffloadBinary. The in-memory layout of the blob is:
+/// [ SymbolTableHeader ]
+/// [ SymbolTableEntry Entries[N] ] -- N == Header.Count
+/// [ char StringData[] ] -- packed null-terminated names
+/// Use writeSymbolTable() to produce the blob and forEachSymbol() to consume
+/// it; both encapsulate all pointer arithmetic.
+
+struct SymbolTableHeader {
+ uint32_t Count; ///< Number of symbol entries.
+};
+struct SymbolTableEntry {
+ uint32_t OffsetToSymbol; ///< Byte offset from blob start to the symbol name.
+ uint32_t SymbolSize; ///< Length of the symbol name in bytes, excluding
+ ///< the null terminator.
+};
+
+/// Serialize \p Names into \p Out.
+LLVM_ABI void writeSymbolTable(ArrayRef<StringRef> Names, SmallString<0> &Out);
+
+/// Invoke \p Callback with a \c StringRef for each symbol in \p Symbols,
+/// the raw serialized symbol-table blob.
+template <typename Fn> void forEachSymbol(StringRef Symbols, Fn &&Callback) {
+ assert(Symbols.size() >= sizeof(SymbolTableHeader) &&
+ "symbols blob smaller than header");
+ const char *Base = Symbols.data();
+ const auto &Header = *reinterpret_cast<const SymbolTableHeader *>(Base);
+ const auto *Entries = reinterpret_cast<const SymbolTableEntry *>(&Header + 1);
+ for (uint32_t I = 0; I < Header.Count; ++I)
+ Callback(
+ StringRef(Base + Entries[I].OffsetToSymbol, Entries[I].SymbolSize));
+}
+
+} // namespace sycl
+
+namespace intel {
+/// Containerizes an OpenMP SPIR-V image into an OffloadBinary image.
+/// \param Binary The SPIR-V binary to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param CompileOpts Optional compilation options.
+/// \param LinkOpts Optional linking options.
+LLVM_ABI Error containerizeOpenMPSPIRVImage(
+ std::unique_ptr<MemoryBuffer> &Binary, llvm::Triple Triple,
+ StringRef CompileOpts = "", StringRef LinkOpts = "");
+} // namespace intel
+} // namespace offloading
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_OFFLOADING_UTILITY_H
diff --git a/llvm/include/llvm/Frontend/Offloading/Utility_BASE_865488.h b/llvm/include/llvm/Frontend/Offloading/Utility_BASE_865488.h
new file mode 100644
index 0000000000000..dad447228d149
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Offloading/Utility_BASE_865488.h
@@ -0,0 +1,231 @@
+//===- Utility.h - Collection of geneirc offloading utilities -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OFFLOADING_UTILITY_H
+#define LLVM_FRONTEND_OFFLOADING_UTILITY_H
+
+#include "llvm/Support/Compiler.h"
+#include <cstdint>
+#include <memory>
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/OffloadBinary.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBufferRef.h"
+
+namespace llvm {
+namespace offloading {
+
+/// This is the record of an object that just be registered with the offloading
+/// runtime.
+struct EntryTy {
+ /// Reserved bytes used to detect an older version of the struct, always zero.
+ uint64_t Reserved = 0x0;
+ /// The current version of the struct for runtime forward compatibility.
+ uint16_t Version = 0x1;
+ /// The expected consumer of this entry, e.g. CUDA or OpenMP.
+ uint16_t Kind;
+ /// Flags associated with the global.
+ uint32_t Flags;
+ /// The address of the global to be registered by the runtime.
+ void *Address;
+ /// The name of the symbol in the device image.
+ char *SymbolName;
+ /// The number of bytes the symbol takes.
+ uint64_t Size;
+ /// Extra generic data used to register this entry.
+ uint64_t Data;
+ /// An extra pointer, usually null.
+ void *AuxAddr;
+};
+
+/// Offloading entry flags for CUDA / HIP. The first three bits indicate the
+/// type of entry while the others are a bit field for additional information.
+enum OffloadEntryKindFlag : uint32_t {
+ /// Mark the entry as a global entry. This indicates the presense of a
+ /// kernel if the size size field is zero and a variable otherwise.
+ OffloadGlobalEntry = 0x0,
+ /// Mark the entry as a managed global variable.
+ OffloadGlobalManagedEntry = 0x1,
+ /// Mark the entry as a surface variable.
+ OffloadGlobalSurfaceEntry = 0x2,
+ /// Mark the entry as a texture variable.
+ OffloadGlobalTextureEntry = 0x3,
+ /// Mark the entry as being extern.
+ OffloadGlobalExtern = 0x1 << 3,
+ /// Mark the entry as being constant.
+ OffloadGlobalConstant = 0x1 << 4,
+ /// Mark the entry as being a normalized surface.
+ OffloadGlobalNormalized = 0x1 << 5,
+};
+
+/// Returns the type of the offloading entry we use to store kernels and
+/// globals that will be registered with the offloading runtime.
+LLVM_ABI StructType *getEntryTy(Module &M);
+
+/// Create an offloading section struct used to register this global at
+/// runtime.
+///
+/// \param M The module to be used
+/// \param Addr The pointer to the global being registered.
+/// \param Kind The offloading language expected to consume this.
+/// \param Name The symbol name associated with the global.
+/// \param Size The size in bytes of the global (0 for functions).
+/// \param Flags Flags associated with the entry.
+/// \param Data Extra data storage associated with the entry.
+/// \param SectionName The section this entry will be placed at.
+/// \param AuxAddr An extra pointer if needed.
+/// \return The emitted global variable containing the offloading entry.
+LLVM_ABI GlobalVariable *
+emitOffloadingEntry(Module &M, object::OffloadKind Kind, Constant *Addr,
+ StringRef Name, uint64_t Size, uint32_t Flags,
+ uint64_t Data, Constant *AuxAddr = nullptr,
+ StringRef SectionName = "llvm_offload_entries");
+
+/// Create a constant struct initializer used to register this global at
+/// runtime.
+/// \return the constant struct and the global variable holding the symbol name.
+LLVM_ABI std::pair<Constant *, GlobalVariable *>
+getOffloadingEntryInitializer(Module &M, object::OffloadKind Kind,
+ Constant *Addr, StringRef Name, uint64_t Size,
+ uint32_t Flags, uint64_t Data, Constant *AuxAddr);
+
+/// Creates a pair of globals used to iterate the array of offloading entries by
+/// accessing the section variables provided by the linker.
+LLVM_ABI std::pair<GlobalVariable *, GlobalVariable *>
+getOffloadEntryArray(Module &M, StringRef SectionName = "llvm_offload_entries");
+
+namespace amdgpu {
+/// Check if an image is compatible with current system's environment. The
+/// system environment is given as a 'target-id' which has the form:
+///
+/// <target-id> := <processor> ( ":" <target-feature> ( "+" | "-" ) )*
+///
+/// If a feature is not specific as '+' or '-' it is assumed to be in an 'any'
+/// and is compatible with either '+' or '-'. The HSA runtime returns this
+/// information using the target-id, while we use the ELF header to determine
+/// these features.
+LLVM_ABI bool isImageCompatibleWithEnv(StringRef ImageArch, uint32_t ImageFlags,
+ StringRef EnvTargetID);
+
+/// Struct for holding metadata related to AMDGPU kernels, for more information
+/// about the metadata and its meaning see:
+/// https://llvm.org/docs/AMDGPUUsage.html#code-object-v3
+struct AMDGPUKernelMetaData {
+ /// Constant indicating that a value is invalid.
+ static constexpr uint32_t KInvalidValue =
+ std::numeric_limits<uint32_t>::max();
+ /// The amount of group segment memory required by a work-group in bytes.
+ uint32_t GroupSegmentList = KInvalidValue;
+ /// The amount of fixed private address space memory required for a work-item
+ /// in bytes.
+ uint32_t PrivateSegmentSize = KInvalidValue;
+ /// Number of scalar registers required by a wavefront.
+ uint32_t SGPRCount = KInvalidValue;
+ /// Number of vector registers required by each work-item.
+ uint32_t VGPRCount = KInvalidValue;
+ /// Number of stores from a scalar register to a register allocator created
+ /// spill location.
+ uint32_t SGPRSpillCount = KInvalidValue;
+ /// Number of stores from a vector register to a register allocator created
+ /// spill location.
+ uint32_t VGPRSpillCount = KInvalidValue;
+ /// Number of accumulator registers required by each work-item.
+ uint32_t AGPRCount = KInvalidValue;
+ /// Corresponds to the OpenCL reqd_work_group_size attribute.
+ uint32_t RequestedWorkgroupSize[3] = {KInvalidValue, KInvalidValue,
+ KInvalidValue};
+ /// Corresponds to the OpenCL work_group_size_hint attribute.
+ uint32_t WorkgroupSizeHint[3] = {KInvalidValue, KInvalidValue, KInvalidValue};
+ /// Wavefront size.
+ uint32_t WavefrontSize = KInvalidValue;
+ /// Maximum flat work-group size supported by the kernel in work-items.
+ uint32_t MaxFlatWorkgroupSize = KInvalidValue;
+};
+
+/// Reads AMDGPU specific metadata from the ELF file and propagates the
+/// KernelInfoMap.
+LLVM_ABI Error getAMDGPUMetaDataFromImage(
+ MemoryBufferRef MemBuffer, StringMap<AMDGPUKernelMetaData> &KernelInfoMap,
+ uint16_t &ELFABIVersion);
+} // namespace amdgpu
+
+/// Containerizes an image within an OffloadBinary image.
+/// Creates a nested OffloadBinary structure where the inner binary contains
+/// the raw image and associated metadata (version, format, triple, etc.).
+/// \param Binary The image to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param ImageKind The format of the image, e.g. SPIR-V or CUBIN.
+/// \param OffloadKind The expected consuming runtime of the image, e.g. CUDA or
+/// OpenMP.
+/// \param ImageFlags Flags associated with the image, e.g. for AMDGPU the
+/// features.
+/// \param MetaData The key-value map of metadata to be associated with the
+/// image.
+LLVM_ABI Error containerizeImage(std::unique_ptr<MemoryBuffer> &Binary,
+ llvm::Triple Triple,
+ object::ImageKind ImageKind,
+ object::OffloadKind OffloadKind,
+ int32_t ImageFlags,
+ MapVector<StringRef, StringRef> &MetaData);
+
+namespace sycl {
+
+/// Serialized symbol table stored in the "symbols" entry of a SYCL
+/// OffloadBinary. The in-memory layout of the blob is:
+/// [ SymbolTableHeader ]
+/// [ SymbolTableEntry Entries[N] ] -- N == Header.Count
+/// [ char StringData[] ] -- packed null-terminated names
+/// Use writeSymbolTable() to produce the blob and forEachSymbol() to consume
+/// it; both encapsulate all pointer arithmetic.
+
+struct SymbolTableHeader {
+ uint32_t Count; ///< Number of symbol entries.
+};
+struct SymbolTableEntry {
+ uint32_t OffsetToSymbol; ///< Byte offset from blob start to the symbol name.
+ uint32_t SymbolSize; ///< Length of the symbol name in bytes, excluding
+ ///< the null terminator.
+};
+
+/// Serialize \p Names into \p Out.
+LLVM_ABI void writeSymbolTable(ArrayRef<StringRef> Names, SmallString<0> &Out);
+
+/// Invoke \p Callback with a \c StringRef for each symbol in \p Symbols,
+/// the raw serialized symbol-table blob.
+template <typename Fn> void forEachSymbol(StringRef Symbols, Fn &&Callback) {
+ assert(Symbols.size() >= sizeof(SymbolTableHeader) &&
+ "symbols blob smaller than header");
+ const char *Base = Symbols.data();
+ const auto &Header = *reinterpret_cast<const SymbolTableHeader *>(Base);
+ const auto *Entries = reinterpret_cast<const SymbolTableEntry *>(&Header + 1);
+ for (uint32_t I = 0; I < Header.Count; ++I)
+ Callback(
+ StringRef(Base + Entries[I].OffsetToSymbol, Entries[I].SymbolSize));
+}
+
+} // namespace sycl
+
+namespace intel {
+/// Containerizes an OpenMP SPIR-V image into an OffloadBinary image.
+/// \param Binary The SPIR-V binary to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param CompileOpts Optional compilation options.
+/// \param LinkOpts Optional linking options.
+LLVM_ABI Error containerizeOpenMPSPIRVImage(
+ std::unique_ptr<MemoryBuffer> &Binary, llvm::Triple Triple,
+ StringRef CompileOpts = "", StringRef LinkOpts = "");
+} // namespace intel
+} // namespace offloading
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_OFFLOADING_UTILITY_H
diff --git a/llvm/include/llvm/Frontend/Offloading/Utility_LOCAL_865488.h b/llvm/include/llvm/Frontend/Offloading/Utility_LOCAL_865488.h
new file mode 100644
index 0000000000000..4c0bc87786dfb
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Offloading/Utility_LOCAL_865488.h
@@ -0,0 +1,240 @@
+//===- Utility.h - Collection of geneirc offloading utilities -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OFFLOADING_UTILITY_H
+#define LLVM_FRONTEND_OFFLOADING_UTILITY_H
+
+#include "llvm/Support/Compiler.h"
+#include <cstdint>
+#include <memory>
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/OffloadBinary.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBufferRef.h"
+
+namespace llvm {
+namespace offloading {
+
+/// This is the record of an object that just be registered with the offloading
+/// runtime.
+struct EntryTy {
+ /// Reserved bytes used to detect an older version of the struct, always zero.
+ uint64_t Reserved = 0x0;
+ /// The current version of the struct for runtime forward compatibility.
+ uint16_t Version = 0x1;
+ /// The expected consumer of this entry, e.g. CUDA or OpenMP.
+ uint16_t Kind;
+ /// Flags associated with the global.
+ uint32_t Flags;
+ /// The address of the global to be registered by the runtime.
+ void *Address;
+ /// The name of the symbol in the device image.
+ char *SymbolName;
+ /// The number of bytes the symbol takes.
+ uint64_t Size;
+ /// Extra generic data used to register this entry.
+ uint64_t Data;
+ /// An extra pointer, usually null.
+ void *AuxAddr;
+};
+
+/// Offloading entry flags for CUDA / HIP. The first three bits indicate the
+/// type of entry while the others are a bit field for additional information.
+enum OffloadEntryKindFlag : uint32_t {
+ /// Mark the entry as a global entry. This indicates the presense of a
+ /// kernel if the size size field is zero and a variable otherwise.
+ OffloadGlobalEntry = 0x0,
+ /// Mark the entry as a managed global variable.
+ OffloadGlobalManagedEntry = 0x1,
+ /// Mark the entry as a surface variable.
+ OffloadGlobalSurfaceEntry = 0x2,
+ /// Mark the entry as a texture variable.
+ OffloadGlobalTextureEntry = 0x3,
+ /// Mark the entry as being extern.
+ OffloadGlobalExtern = 0x1 << 3,
+ /// Mark the entry as being constant.
+ OffloadGlobalConstant = 0x1 << 4,
+ /// Mark the entry as being a normalized surface.
+ OffloadGlobalNormalized = 0x1 << 5,
+};
+
+/// Returns the type of the offloading entry we use to store kernels and
+/// globals that will be registered with the offloading runtime.
+LLVM_ABI StructType *getEntryTy(Module &M);
+
+/// Create an offloading section struct used to register this global at
+/// runtime.
+///
+/// \param M The module to be used
+/// \param Addr The pointer to the global being registered.
+/// \param Kind The offloading language expected to consume this.
+/// \param Name The symbol name associated with the global.
+/// \param Size The size in bytes of the global (0 for functions).
+/// \param Flags Flags associated with the entry.
+/// \param Data Extra data storage associated with the entry.
+/// \param SectionName The section this entry will be placed at.
+/// \param AuxAddr An extra pointer if needed.
+/// Returns the section name for offloading entries based on the target triple.
+/// ELF: "llvm_offload_entries", COFF: "llvm_offload_entries",
+/// Mach-O: "__LLVM,offload_entries".
+LLVM_ABI StringRef getOffloadEntrySection(Module &M);
+
+/// \return The emitted global variable containing the offloading entry.
+LLVM_ABI GlobalVariable *
+emitOffloadingEntry(Module &M, object::OffloadKind Kind, Constant *Addr,
+ StringRef Name, uint64_t Size, uint32_t Flags,
+ uint64_t Data, Constant *AuxAddr = nullptr);
+
+/// Create a constant struct initializer used to register this global at
+/// runtime.
+/// \return the constant struct and the global variable holding the symbol name.
+LLVM_ABI std::pair<Constant *, GlobalVariable *>
+getOffloadingEntryInitializer(Module &M, object::OffloadKind Kind,
+ Constant *Addr, StringRef Name, uint64_t Size,
+ uint32_t Flags, uint64_t Data, Constant *AuxAddr);
+
+/// Creates a pair of globals used to iterate the array of offloading entries by
+/// accessing the section variables provided by the linker.
+LLVM_ABI std::pair<GlobalVariable *, GlobalVariable *>
+getOffloadEntryArray(Module &M);
+
+namespace amdgpu {
+/// Check if an image is compatible with current system's environment. The
+/// system environment is given as a 'target-id' which has the form:
+///
+/// <target-id> := <processor> ( ":" <target-feature> ( "+" | "-" ) )*
+///
+/// If a feature is not specific as '+' or '-' it is assumed to be in an 'any'
+/// and is compatible with either '+' or '-'. The HSA runtime returns this
+/// information using the target-id, while we use the ELF header to determine
+/// these features.
+LLVM_ABI bool isImageCompatibleWithEnv(StringRef ImageArch, uint32_t ImageFlags,
+ StringRef EnvTargetID);
+
+/// Struct for holding metadata related to AMDGPU kernels, for more information
+/// about the metadata and its meaning see:
+/// https://llvm.org/docs/AMDGPUUsage.html#code-object-v3
+struct AMDGPUKernelMetaData {
+ /// Constant indicating that a value is invalid.
+ static constexpr uint32_t KInvalidValue =
+ std::numeric_limits<uint32_t>::max();
+ /// The amount of group segment memory required by a work-group in bytes.
+ uint32_t GroupSegmentList = KInvalidValue;
+ /// The amount of fixed private address space memory required for a work-item
+ /// in bytes.
+ uint32_t PrivateSegmentSize = KInvalidValue;
+ /// Number of scalar registers required by a wavefront.
+ uint32_t SGPRCount = KInvalidValue;
+ /// Number of vector registers required by each work-item.
+ uint32_t VGPRCount = KInvalidValue;
+ /// Number of stores from a scalar register to a register allocator created
+ /// spill location.
+ uint32_t SGPRSpillCount = KInvalidValue;
+ /// Number of stores from a vector register to a register allocator created
+ /// spill location.
+ uint32_t VGPRSpillCount = KInvalidValue;
+ /// Number of accumulator registers required by each work-item.
+ uint32_t AGPRCount = KInvalidValue;
+ /// Corresponds to the OpenCL reqd_work_group_size attribute.
+ uint32_t RequestedWorkgroupSize[3] = {KInvalidValue, KInvalidValue,
+ KInvalidValue};
+ /// Corresponds to the OpenCL work_group_size_hint attribute.
+ uint32_t WorkgroupSizeHint[3] = {KInvalidValue, KInvalidValue, KInvalidValue};
+ /// Wavefront size.
+ uint32_t WavefrontSize = KInvalidValue;
+ /// Maximum flat work-group size supported by the kernel in work-items.
+ uint32_t MaxFlatWorkgroupSize = KInvalidValue;
+ /// Per-argument {offset, size} in bytes, read from the ".args" array in code
+ /// object metadata. Explicit user arguments are first, followed by
+ /// hidden arguments.
+ SmallVector<std::pair<uint32_t, uint32_t>, 8> ArgMDs;
+};
+
+/// Reads AMDGPU specific metadata from the ELF file and propagates the
+/// KernelInfoMap.
+LLVM_ABI Error getAMDGPUMetaDataFromImage(
+ MemoryBufferRef MemBuffer, StringMap<AMDGPUKernelMetaData> &KernelInfoMap,
+ uint16_t &ELFABIVersion);
+} // namespace amdgpu
+
+/// Containerizes an image within an OffloadBinary image.
+/// Creates a nested OffloadBinary structure where the inner binary contains
+/// the raw image and associated metadata (version, format, triple, etc.).
+/// \param Binary The image to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param ImageKind The format of the image, e.g. SPIR-V or CUBIN.
+/// \param OffloadKind The expected consuming runtime of the image, e.g. CUDA or
+/// OpenMP.
+/// \param ImageFlags Flags associated with the image, e.g. for AMDGPU the
+/// features.
+/// \param MetaData The key-value map of metadata to be associated with the
+/// image.
+LLVM_ABI Error containerizeImage(std::unique_ptr<MemoryBuffer> &Binary,
+ llvm::Triple Triple,
+ object::ImageKind ImageKind,
+ object::OffloadKind OffloadKind,
+ int32_t ImageFlags,
+ MapVector<StringRef, StringRef> &MetaData);
+
+namespace sycl {
+
+/// Serialized symbol table stored in the "symbols" entry of a SYCL
+/// OffloadBinary. The in-memory layout of the blob is:
+/// [ SymbolTableHeader ]
+/// [ SymbolTableEntry Entries[N] ] -- N == Header.Count
+/// [ char StringData[] ] -- packed null-terminated names
+/// Use writeSymbolTable() to produce the blob and forEachSymbol() to consume
+/// it; both encapsulate all pointer arithmetic.
+
+struct SymbolTableHeader {
+ uint32_t Count; ///< Number of symbol entries.
+};
+struct SymbolTableEntry {
+ uint32_t OffsetToSymbol; ///< Byte offset from blob start to the symbol name.
+ uint32_t SymbolSize; ///< Length of the symbol name in bytes, excluding
+ ///< the null terminator.
+};
+
+/// Serialize \p Names into \p Out.
+LLVM_ABI void writeSymbolTable(ArrayRef<StringRef> Names, SmallString<0> &Out);
+
+/// Invoke \p Callback with a \c StringRef for each symbol in \p Symbols,
+/// the raw serialized symbol-table blob.
+template <typename Fn> void forEachSymbol(StringRef Symbols, Fn &&Callback) {
+ assert(Symbols.size() >= sizeof(SymbolTableHeader) &&
+ "symbols blob smaller than header");
+ const char *Base = Symbols.data();
+ const auto &Header = *reinterpret_cast<const SymbolTableHeader *>(Base);
+ const auto *Entries = reinterpret_cast<const SymbolTableEntry *>(&Header + 1);
+ for (uint32_t I = 0; I < Header.Count; ++I)
+ Callback(
+ StringRef(Base + Entries[I].OffsetToSymbol, Entries[I].SymbolSize));
+}
+
+} // namespace sycl
+
+namespace intel {
+/// Containerizes an OpenMP SPIR-V image into an OffloadBinary image.
+/// \param Binary The SPIR-V binary to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param CompileOpts Optional compilation options.
+/// \param LinkOpts Optional linking options.
+LLVM_ABI Error containerizeOpenMPSPIRVImage(
+ std::unique_ptr<MemoryBuffer> &Binary, llvm::Triple Triple,
+ StringRef CompileOpts = "", StringRef LinkOpts = "");
+} // namespace intel
+} // namespace offloading
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_OFFLOADING_UTILITY_H
diff --git a/llvm/include/llvm/Frontend/Offloading/Utility_REMOTE_865488.h b/llvm/include/llvm/Frontend/Offloading/Utility_REMOTE_865488.h
new file mode 100644
index 0000000000000..36f4fbc713ada
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Offloading/Utility_REMOTE_865488.h
@@ -0,0 +1,247 @@
+//===- Utility.h - Collection of geneirc offloading utilities -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OFFLOADING_UTILITY_H
+#define LLVM_FRONTEND_OFFLOADING_UTILITY_H
+
+#include "llvm/Support/Compiler.h"
+#include <cstdint>
+#include <memory>
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/OffloadBinary.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBufferRef.h"
+
+namespace llvm {
+namespace offloading {
+
+/// This is the record of an object that just be registered with the offloading
+/// runtime.
+struct EntryTy {
+ /// Reserved bytes used to detect an older version of the struct, always zero.
+ uint64_t Reserved = 0x0;
+ /// The current version of the struct for runtime forward compatibility.
+ uint16_t Version = 0x1;
+ /// The expected consumer of this entry, e.g. CUDA or OpenMP.
+ uint16_t Kind;
+ /// Flags associated with the global.
+ uint32_t Flags;
+ /// The address of the global to be registered by the runtime.
+ void *Address;
+ /// The name of the symbol in the device image.
+ char *SymbolName;
+ /// The number of bytes the symbol takes.
+ uint64_t Size;
+ /// Extra generic data used to register this entry.
+ uint64_t Data;
+ /// An extra pointer, usually null.
+ void *AuxAddr;
+};
+
+/// Offloading entry flags for CUDA / HIP. The first three bits indicate the
+/// type of entry while the others are a bit field for additional information.
+enum OffloadEntryKindFlag : uint32_t {
+ /// Mark the entry as a global entry. This indicates the presense of a
+ /// kernel if the size size field is zero and a variable otherwise.
+ OffloadGlobalEntry = 0x0,
+ /// Mark the entry as a managed global variable.
+ OffloadGlobalManagedEntry = 0x1,
+ /// Mark the entry as a surface variable.
+ OffloadGlobalSurfaceEntry = 0x2,
+ /// Mark the entry as a texture variable.
+ OffloadGlobalTextureEntry = 0x3,
+ /// Mark the entry as being extern.
+ OffloadGlobalExtern = 0x1 << 3,
+ /// Mark the entry as being constant.
+ OffloadGlobalConstant = 0x1 << 4,
+ /// Mark the entry as being a normalized surface.
+ OffloadGlobalNormalized = 0x1 << 5,
+};
+
+/// Returns the type of the offloading entry we use to store kernels and
+/// globals that will be registered with the offloading runtime.
+LLVM_ABI StructType *getEntryTy(Module &M);
+
+/// Create an offloading section struct used to register this global at
+/// runtime.
+///
+/// \param M The module to be used
+/// \param Addr The pointer to the global being registered.
+/// \param Kind The offloading language expected to consume this.
+/// \param Name The symbol name associated with the global.
+/// \param Size The size in bytes of the global (0 for functions).
+/// \param Flags Flags associated with the entry.
+/// \param Data Extra data storage associated with the entry.
+/// \param SectionName The section this entry will be placed at.
+/// \param AuxAddr An extra pointer if needed.
+/// \return The emitted global variable containing the offloading entry.
+LLVM_ABI GlobalVariable *
+emitOffloadingEntry(Module &M, object::OffloadKind Kind, Constant *Addr,
+ StringRef Name, uint64_t Size, uint32_t Flags,
+ uint64_t Data, Constant *AuxAddr = nullptr,
+ StringRef SectionName = "llvm_offload_entries");
+
+/// Create a constant struct initializer used to register this global at
+/// runtime.
+/// \return the constant struct and the global variable holding the symbol name.
+LLVM_ABI std::pair<Constant *, GlobalVariable *>
+getOffloadingEntryInitializer(Module &M, object::OffloadKind Kind,
+ Constant *Addr, StringRef Name, uint64_t Size,
+ uint32_t Flags, uint64_t Data, Constant *AuxAddr);
+
+/// Creates a pair of globals used to iterate the array of offloading entries by
+/// accessing the section variables provided by the linker.
+LLVM_ABI std::pair<GlobalVariable *, GlobalVariable *>
+getOffloadEntryArray(Module &M, StringRef SectionName = "llvm_offload_entries");
+
+namespace amdgpu {
+/// Check if an image is compatible with current system's environment. The
+/// system environment is given as a 'target-id' which has the form:
+///
+/// <target-id> := <processor> ( ":" <target-feature> ( "+" | "-" ) )*
+///
+/// If a feature is not specific as '+' or '-' it is assumed to be in an 'any'
+/// and is compatible with either '+' or '-'. The HSA runtime returns this
+/// information using the target-id, while we use the ELF header to determine
+/// these features.
+LLVM_ABI bool isImageCompatibleWithEnv(StringRef ImageArch, uint32_t ImageFlags,
+ StringRef EnvTargetID);
+
+/// The offset and size of an AMD GPU kernel argument, as read from MetaData.
+struct AMDGPUKernelArgumentInfo {
+ // Address space and value kind are available but not stored right now.
+ uint32_t Offset;
+ uint32_t Size;
+};
+
+/// Container for the kernel argument layout, as read from MetaData.
+struct AMDGPUKernelArgumentLayout {
+ SmallVector<AMDGPUKernelArgumentInfo, 32> Arguments;
+ int32_t NumUserArguments = -1;
+};
+
+/// Struct for holding metadata related to AMDGPU kernels, for more information
+/// about the metadata and its meaning see:
+/// https://llvm.org/docs/AMDGPUUsage.html#code-object-v3
+struct AMDGPUKernelMetaData {
+ /// Constant indicating that a value is invalid.
+ static constexpr uint32_t KInvalidValue =
+ std::numeric_limits<uint32_t>::max();
+ /// The amount of group segment memory required by a work-group in bytes.
+ uint32_t GroupSegmentList = KInvalidValue;
+ /// The amount of fixed private address space memory required for a work-item
+ /// in bytes.
+ uint32_t PrivateSegmentSize = KInvalidValue;
+ /// Number of scalar registers required by a wavefront.
+ uint32_t SGPRCount = KInvalidValue;
+ /// Number of vector registers required by each work-item.
+ uint32_t VGPRCount = KInvalidValue;
+ /// Number of stores from a scalar register to a register allocator created
+ /// spill location.
+ uint32_t SGPRSpillCount = KInvalidValue;
+ /// Number of stores from a vector register to a register allocator created
+ /// spill location.
+ uint32_t VGPRSpillCount = KInvalidValue;
+ /// Number of accumulator registers required by each work-item.
+ uint32_t AGPRCount = KInvalidValue;
+ /// Corresponds to the OpenCL reqd_work_group_size attribute.
+ uint32_t RequestedWorkgroupSize[3] = {KInvalidValue, KInvalidValue,
+ KInvalidValue};
+ /// Corresponds to the OpenCL work_group_size_hint attribute.
+ uint32_t WorkgroupSizeHint[3] = {KInvalidValue, KInvalidValue, KInvalidValue};
+ /// Wavefront size.
+ uint32_t WavefrontSize = KInvalidValue;
+ /// Maximum flat work-group size supported by the kernel in work-items.
+ uint32_t MaxFlatWorkgroupSize = KInvalidValue;
+ /// The argument layout of this kernel.
+ AMDGPUKernelArgumentLayout ArgumentLayout;
+};
+
+/// Reads AMDGPU specific metadata from the ELF file and propagates the
+/// KernelInfoMap.
+LLVM_ABI Error getAMDGPUMetaDataFromImage(
+ MemoryBufferRef MemBuffer, StringMap<AMDGPUKernelMetaData> &KernelInfoMap,
+ uint16_t &ELFABIVersion);
+} // namespace amdgpu
+
+/// Containerizes an image within an OffloadBinary image.
+/// Creates a nested OffloadBinary structure where the inner binary contains
+/// the raw image and associated metadata (version, format, triple, etc.).
+/// \param Binary The image to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param ImageKind The format of the image, e.g. SPIR-V or CUBIN.
+/// \param OffloadKind The expected consuming runtime of the image, e.g. CUDA or
+/// OpenMP.
+/// \param ImageFlags Flags associated with the image, e.g. for AMDGPU the
+/// features.
+/// \param MetaData The key-value map of metadata to be associated with the
+/// image.
+LLVM_ABI Error containerizeImage(std::unique_ptr<MemoryBuffer> &Binary,
+ llvm::Triple Triple,
+ object::ImageKind ImageKind,
+ object::OffloadKind OffloadKind,
+ int32_t ImageFlags,
+ MapVector<StringRef, StringRef> &MetaData);
+
+namespace sycl {
+
+/// Serialized symbol table stored in the "symbols" entry of a SYCL
+/// OffloadBinary. The in-memory layout of the blob is:
+/// [ SymbolTableHeader ]
+/// [ SymbolTableEntry Entries[N] ] -- N == Header.Count
+/// [ char StringData[] ] -- packed null-terminated names
+/// Use writeSymbolTable() to produce the blob and forEachSymbol() to consume
+/// it; both encapsulate all pointer arithmetic.
+
+struct SymbolTableHeader {
+ uint32_t Count; ///< Number of symbol entries.
+};
+struct SymbolTableEntry {
+ uint32_t OffsetToSymbol; ///< Byte offset from blob start to the symbol name.
+ uint32_t SymbolSize; ///< Length of the symbol name in bytes, excluding
+ ///< the null terminator.
+};
+
+/// Serialize \p Names into \p Out.
+LLVM_ABI void writeSymbolTable(ArrayRef<StringRef> Names, SmallString<0> &Out);
+
+/// Invoke \p Callback with a \c StringRef for each symbol in \p Symbols,
+/// the raw serialized symbol-table blob.
+template <typename Fn> void forEachSymbol(StringRef Symbols, Fn &&Callback) {
+ assert(Symbols.size() >= sizeof(SymbolTableHeader) &&
+ "symbols blob smaller than header");
+ const char *Base = Symbols.data();
+ const auto &Header = *reinterpret_cast<const SymbolTableHeader *>(Base);
+ const auto *Entries = reinterpret_cast<const SymbolTableEntry *>(&Header + 1);
+ for (uint32_t I = 0; I < Header.Count; ++I)
+ Callback(
+ StringRef(Base + Entries[I].OffsetToSymbol, Entries[I].SymbolSize));
+}
+
+} // namespace sycl
+
+namespace intel {
+/// Containerizes an OpenMP SPIR-V image into an OffloadBinary image.
+/// \param Binary The SPIR-V binary to containerize.
+/// \param Triple The target triple to be associated with the image.
+/// \param CompileOpts Optional compilation options.
+/// \param LinkOpts Optional linking options.
+LLVM_ABI Error containerizeOpenMPSPIRVImage(
+ std::unique_ptr<MemoryBuffer> &Binary, llvm::Triple Triple,
+ StringRef CompileOpts = "", StringRef LinkOpts = "");
+} // namespace intel
+} // namespace offloading
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_OFFLOADING_UTILITY_H
diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h
index 1a5a9305bbda6..8b080aca69070 100644
--- a/llvm/include/llvm/Object/OffloadBinary.h
+++ b/llvm/include/llvm/Object/OffloadBinary.h
@@ -37,7 +37,8 @@ enum OffloadKind : uint16_t {
OFK_Cuda = (1 << 1),
OFK_HIP = (1 << 2),
OFK_SYCL = (1 << 3),
- OFK_LAST = (1 << 4),
+ OFK_LLVM = OFK_OpenMP | OFK_Cuda | OFK_HIP,
+ OFK_LAST = (1 << 5),
};
/// The type of contents the offloading image contains.
diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
index ded603a1e00e3..28f6a6a38e751 100644
--- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
+++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
@@ -485,10 +485,12 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
llvm::offloading::OffloadGlobalNormalized));
auto *Normalized = Builder.CreateLShr(
NormalizedBit, ConstantInt::get(Type::getInt32Ty(C), 5), "normalized");
- auto *KindCond = Builder.CreateICmpEQ(
+ auto *KindAnd = Builder.CreateAnd(
Kind, ConstantInt::get(Type::getInt16Ty(C),
IsHIP ? object::OffloadKind::OFK_HIP
: object::OffloadKind::OFK_Cuda));
+ auto *KindCond = Builder.CreateICmpNE(
+ KindAnd, ConstantInt::get(Type::getInt16Ty(C), 0));
Builder.CreateCondBr(KindCond, IfKindBB, IfEndBB);
Builder.SetInsertPoint(IfKindBB);
auto *FnCond = Builder.CreateICmpEQ(
diff --git a/llvm/llvm.code-workspace b/llvm/llvm.code-workspace
new file mode 100644
index 0000000000000..8f33bbe791b86
--- /dev/null
+++ b/llvm/llvm.code-workspace
@@ -0,0 +1,23 @@
+{
+ "folders": [
+ {
+ "path": "."
+ },
+ {
+ "path": "../flang"
+ },
+ {
+ "path": "../flang-rt"
+ },
+ {
+ "path": "../clang"
+ },
+ {
+ "path": "../offload"
+ },
+ {
+ "path": "../openmp"
+ }
+ ],
+ "settings": {}
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-no-kernel.ll.kernels.inc b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-no-kernel.ll.kernels.inc
new file mode 100644
index 0000000000000..080ea5a650362
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-no-kernel.ll.kernels.inc
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-attributor %s | FileCheck %s --check-prefix WITHKERNEL
+
+declare void @caller_wgs()
+declare void @caller_wpe()
+declare void @caller_wgs_wpe()
+
+define amdgpu_kernel void @kernel_256_512() #0 {
+ call void @caller_wgs()
+ call void @caller_wpe()
+ call void @caller_wgs_wpe()
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="256,512" }
diff --git a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-with-kernel.ll b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-with-kernel.ll
new file mode 100644
index 0000000000000..5383995b8009e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu-with-kernel.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor %s | FileCheck %s
+
+define amdgpu_kernel void @kernel_512_512() #3 {
+; CHECK-LABEL: define {{[^@]+}}@kernel_512_512
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: call void @caller_wgs()
+; CHECK-NEXT: call void @caller_wpe()
+; CHECK-NEXT: call void @caller_wgs_wpe()
+; CHECK-NEXT: ret void
+;
+ call void @caller_wgs()
+ call void @caller_wpe()
+ call void @caller_wgs_wpe()
+ ret void
+}
+
+define private void @callee_wgs() #0 {
+; CHECK-LABEL: define {{[^@]+}}@callee_wgs
+; CHECK-SAME: () #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define private void @caller_wgs() #0 {
+; CHECK-LABEL: define {{[^@]+}}@caller_wgs
+; CHECK-SAME: () #[[ATTR1]] {
+; CHECK-NEXT: call void @callee_wgs()
+; CHECK-NEXT: ret void
+;
+ call void @callee_wgs()
+ ret void
+}
+
+define private void @callee_wpe() #1 {
+; CHECK-LABEL: define {{[^@]+}}@callee_wpe
+; CHECK-SAME: () #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define private void @caller_wpe() #1 {
+; CHECK-LABEL: define {{[^@]+}}@caller_wpe
+; CHECK-SAME: () #[[ATTR2]] {
+; CHECK-NEXT: call void @callee_wpe()
+; CHECK-NEXT: ret void
+;
+ call void @callee_wpe()
+ ret void
+}
+
+define private void @callee_wgs_wpe() #2 {
+; CHECK-LABEL: define {{[^@]+}}@callee_wgs_wpe
+; CHECK-SAME: () #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define private void @caller_wgs_wpe() #2 {
+; CHECK-LABEL: define {{[^@]+}}@caller_wgs_wpe
+; CHECK-SAME: () #[[ATTR3]] {
+; CHECK-NEXT: call void @callee_wgs_wpe()
+; CHECK-NEXT: ret void
+;
+ call void @callee_wgs_wpe()
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="128,640" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+attributes #1 = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,6" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+attributes #2 = { "amdgpu-flat-work-group-size"="256,768" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,6" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+attributes #3 = { "amdgpu-flat-work-group-size"="512,512" }
+;.
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,6" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,6" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+;.
diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index b218ed264707c..b2aaa26c5676e 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -349,6 +349,7 @@ if(BUILD_LIBOMPTARGET)
endif()
add_subdirectory(liboffload)
+add_subdirectory(languages)
# Add tests.
if(OFFLOAD_INCLUDE_TESTS)
diff --git a/offload/languages/CMakeLists.txt b/offload/languages/CMakeLists.txt
new file mode 100644
index 0000000000000..fbd774e645e21
--- /dev/null
+++ b/offload/languages/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_subdirectory(kernel)
+add_subdirectory(cuda)
+add_subdirectory(hip)
diff --git a/offload/languages/cuda/CMakeLists.txt b/offload/languages/cuda/CMakeLists.txt
new file mode 100644
index 0000000000000..5e9dda6a8b49e
--- /dev/null
+++ b/offload/languages/cuda/CMakeLists.txt
@@ -0,0 +1,42 @@
+add_llvm_library(
+ LLVMcudart SHARED
+
+ src/cuda_runtime.cpp
+
+ LINK_COMPONENTS
+ Support
+ Offload
+ OffloadKernel
+ )
+
+if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+ target_link_libraries(LLVMcudart PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+endif()
+
+target_include_directories(LLVMcudart PUBLIC
+ ${CMAKE_CURRENT_SOURCE_DIR}/../include/cuda
+ ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel
+ ${CMAKE_CURRENT_SOURCE_DIR}/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../kernel/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include/generated
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../plugins-nextgen/common/include)
+
+target_compile_options(LLVMcudart PRIVATE ${offload_compile_flags} -fdeclspec)
+target_link_options(LLVMcudart PRIVATE ${offload_link_flags})
+
+target_compile_definitions(LLVMcudart PRIVATE
+ TARGET_NAME="libLLVMcudart"
+ DEBUG_PREFIX="LLVMcudart"
+)
+
+set_target_properties(LLVMcudart PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY "${LLVM_LIBRARY_OUTPUT_INTDIR}/${OFFLOAD_TARGET_SUBDIR}"
+ POSITION_INDEPENDENT_CODE ON
+ INSTALL_RPATH "$ORIGIN"
+ BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
+install(TARGETS LLVMcudart ARCHIVE DESTINATION "${OFFLOAD_INSTALL_LIBDIR}"
+ COMPONENT offload)
+
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/cuda/cuda_runtime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/cuda/)
diff --git a/offload/languages/cuda/exports b/offload/languages/cuda/exports
new file mode 100644
index 0000000000000..0851a8545ad2d
--- /dev/null
+++ b/offload/languages/cuda/exports
@@ -0,0 +1,10 @@
+VERS1.0 {
+ global:
+ *cuda*;
+ llvmLaunchKernel*;
+ __llvm*;
+ __tgt_register_lib;
+ __tgt_unregister_lib;
+ local:
+ *;
+};
diff --git a/offload/languages/cuda/src/cuda_runtime.cpp b/offload/languages/cuda/src/cuda_runtime.cpp
new file mode 100644
index 0000000000000..4472e74ad529c
--- /dev/null
+++ b/offload/languages/cuda/src/cuda_runtime.cpp
@@ -0,0 +1,27 @@
+/*===---- cuda_runtime.cpp - CUDA runtime api implementations --------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#include "cuda_runtime.h"
+
+#include "OffloadAPI.h"
+
+#define LANGUAGE cuda
+
+#include "../../kernel/src/LanguageRuntime.cpp"
+
+#include "../../kernel/src/LanguageRegistration.cpp"
+
+#include "../../kernel/src/LanguageLaunch.cpp"
+
+// Must be last as it introduces alises for some definitions from above.
+#include "LanguageAliases.h"
+
+extern "C" {
+void __cudaRegisterFatBinaryEnd(void *) {}
+}
diff --git a/offload/languages/hip/CMakeLists.txt b/offload/languages/hip/CMakeLists.txt
new file mode 100644
index 0000000000000..ec56a76a0475e
--- /dev/null
+++ b/offload/languages/hip/CMakeLists.txt
@@ -0,0 +1,42 @@
+add_llvm_library(
+ LLVMhip64 SHARED
+
+ src/hip_runtime.cpp
+
+ LINK_COMPONENTS
+ Support
+ Offload
+ OffloadKernel
+ )
+
+if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+ target_link_libraries(LLVMhip64 PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+endif()
+
+target_include_directories(LLVMhip64 PUBLIC
+ ${CMAKE_CURRENT_SOURCE_DIR}/../include/hip
+ ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel
+ ${CMAKE_CURRENT_SOURCE_DIR}/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../kernel/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include/generated
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../plugins-nextgen/common/include)
+
+target_compile_options(LLVMhip64 PRIVATE ${offload_compile_flags} -fdeclspec)
+target_link_options(LLVMhip64 PRIVATE ${offload_link_flags})
+
+target_compile_definitions(LLVMhip64 PRIVATE
+ TARGET_NAME="libLLVMhiprt"
+ DEBUG_PREFIX="LLVMhip64"
+)
+
+set_target_properties(LLVMhip64 PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY "${LLVM_LIBRARY_OUTPUT_INTDIR}/${OFFLOAD_TARGET_SUBDIR}"
+ POSITION_INDEPENDENT_CODE ON
+ INSTALL_RPATH "$ORIGIN"
+ BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
+install(TARGETS LLVMhip64 ARCHIVE DESTINATION "${OFFLOAD_INSTALL_LIBDIR}"
+ COMPONENT offload)
+
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/hip/hip_runtime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/hip/)
diff --git a/offload/languages/hip/exports b/offload/languages/hip/exports
new file mode 100644
index 0000000000000..43c36bb1543e9
--- /dev/null
+++ b/offload/languages/hip/exports
@@ -0,0 +1,10 @@
+VERS1.0 {
+ global:
+ *hip*;
+ llvmLaunchKernel*;
+ __llvm*;
+ __tgt_register_lib;
+ __tgt_unregister_lib;
+ local:
+ *;
+};
diff --git a/offload/languages/hip/src/hip_runtime.cpp b/offload/languages/hip/src/hip_runtime.cpp
new file mode 100644
index 0000000000000..34738cb09da7f
--- /dev/null
+++ b/offload/languages/hip/src/hip_runtime.cpp
@@ -0,0 +1,31 @@
+/*===---- hip_runtime.cpp - HIP runtime api implementations ----------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#include "hip_runtime.h"
+
+#include "OffloadAPI.h"
+
+#define LANGUAGE hip
+
+#include "../../kernel/src/LanguageRuntime.cpp"
+
+#include "../../kernel/src/LanguageRegistration.cpp"
+
+#include "../../kernel/src/LanguageLaunch.cpp"
+
+// Must be last as it introduces alises for some definitions from above.
+#include "LanguageAliases.h"
+
+extern "C" hipError_t hipLaunchKernel(const char *KernelID, dim3 GridDim,
+ dim3 BlockDim, void **KernelArgsPtr,
+ size_t DynamicSharedMem, void *Stream) {
+ return olKConvertResult(
+ __llvmLaunchKernelImpl(KernelID, GridDim, BlockDim, KernelArgsPtr,
+ DynamicSharedMem, Stream));
+}
diff --git a/offload/languages/include/cuda/cuda_runtime.h b/offload/languages/include/cuda/cuda_runtime.h
new file mode 100644
index 0000000000000..4594784b0b8a1
--- /dev/null
+++ b/offload/languages/include/cuda/cuda_runtime.h
@@ -0,0 +1,22 @@
+/*===---- cuda_runtime.h - CUDA runtime api declarations -------------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#pragma once
+
+#define LANGUAGE cuda
+
+#include "../kernel/DefineLanguageNames.inc"
+
+#include "../kernel/LanguageRuntime.h"
+
+#include "../kernel/UndefineLanguageNames.inc"
+
+#undef LANGUAGE
+
+using cudaDeviceProp = cudaDeviceProp_t;
diff --git a/offload/languages/include/hip/hip_runtime.h b/offload/languages/include/hip/hip_runtime.h
new file mode 100644
index 0000000000000..9762a8a797d34
--- /dev/null
+++ b/offload/languages/include/hip/hip_runtime.h
@@ -0,0 +1,47 @@
+/*===---- hip_runtime.h - HIP runtime api declarations ---------------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#pragma once
+
+#define LANGUAGE hip
+
+#include "../kernel/DefineLanguageNames.inc"
+
+#include "../kernel/LanguageRuntime.h"
+
+#include "../kernel/UndefineLanguageNames.inc"
+
+#undef LANGUAGE
+
+enum hipHostMallocFlag_t { hipHostMallocNonCoherent = 0x80000000 };
+
+hipError_t hipHostMalloc(void **Ptr, size_t Size, unsigned int Flags) {
+ return hipHostAlloc(Ptr, Size, Flags);
+}
+
+template <class T>
+static inline hipError_t hipHostMalloc(T **Ptr, size_t Size,
+ unsigned int Flags) {
+ return ::hipHostMalloc((void **)Ptr, Size, Flags);
+}
+
+#if defined(__AMDGPU__) || defined(__NVPTX__)
+#define HIP_KERNEL_NAME(...) __VA_ARGS__
+
+extern "C" hipError_t hipLaunchKernel(const char *Kernel, dim3 GridDim,
+ dim3 BlockDim, void **KernelArgs,
+ size_t DynamicSharedMem, void *Stream);
+
+template <typename... AT, typename FT = void (*)(AT...)>
+static inline void hipLaunchKernelGGL(FT Kernel, dim3 GridDim, dim3 BlockDim,
+ size_t DynamicSharedMem, void *Stream,
+ AT... KernelArgs) {
+ Kernel<<<GridDim, BlockDim, DynamicSharedMem, Stream>>>(KernelArgs...);
+}
+#endif
diff --git a/offload/languages/include/kernel/DefineLanguageNames.inc b/offload/languages/include/kernel/DefineLanguageNames.inc
new file mode 100644
index 0000000000000..80dc532589ad5
--- /dev/null
+++ b/offload/languages/include/kernel/DefineLanguageNames.inc
@@ -0,0 +1,37 @@
+//===-- LanguageNames.inc - Kernel Language runtime API names -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#define COMBINE2(X, Y) X##Y
+#define COMBINE(X, Y) COMBINE2(X, Y)
+
+#define Error_t COMBINE(LANGUAGE, Error_t)
+#define DeviceProp_t COMBINE(LANGUAGE, DeviceProp_t)
+#define Malloc COMBINE(LANGUAGE, Malloc)
+#define Free COMBINE(LANGUAGE, Free)
+#define Memcpy COMBINE(LANGUAGE, Memcpy)
+#define DeviceSynchronize COMBINE(LANGUAGE, DeviceSynchronize)
+#define Success COMBINE(LANGUAGE, Success)
+#define MemcpyKind COMBINE(LANGUAGE, MemcpyKind)
+#define MemcpyHostToHost COMBINE(LANGUAGE, MemcpyHostToHost)
+#define MemcpyHostToDevice COMBINE(LANGUAGE, MemcpyHostToDevice)
+#define MemcpyDeviceToHost COMBINE(LANGUAGE, MemcpyDeviceToHost)
+#define MemcpyDeviceToDevice COMBINE(LANGUAGE, MemcpyDeviceToDevice)
+#define MemcpyDefault COMBINE(LANGUAGE, MemcpyDefault)
+#define GetLastError COMBINE(LANGUAGE, GetLastError)
+#define PeekAtLastError COMBINE(LANGUAGE, PeekAtLastError)
+#define GetErrorName COMBINE(LANGUAGE, GetErrorName)
+#define GetErrorString COMBINE(LANGUAGE, GetErrorString)
+#define GetDeviceCount COMBINE(LANGUAGE, GetDeviceCount)
+#define SetDevice COMBINE(LANGUAGE, SetDevice)
+#define HostAlloc COMBINE(LANGUAGE, HostAlloc)
+#define MallocHost COMBINE(LANGUAGE, MallocHost)
+#define HostFree COMBINE(LANGUAGE, HostFree)
+#define DriverGetVersion COMBINE(LANGUAGE, DriverGetVersion)
+#define GetDeviceProperties COMBINE(LANGUAGE, GetDeviceProperties)
diff --git a/offload/languages/include/kernel/LanguageRuntime.h b/offload/languages/include/kernel/LanguageRuntime.h
new file mode 100644
index 0000000000000..483eb33acb62f
--- /dev/null
+++ b/offload/languages/include/kernel/LanguageRuntime.h
@@ -0,0 +1,165 @@
+/*===---- language_runtime.h - Kernel language runtime api declarations ----===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+
+enum Error_t : uint32_t {
+ Success = 0,
+};
+
+struct DeviceProp_t {
+ char name[256];
+ size_t totalGlobalMem;
+ int multiProcessorCount;
+ int major;
+ int minor;
+};
+
+enum MemcpyKind {
+ MemcpyHostToHost = 0,
+ MemcpyHostToDevice = 1,
+ MemcpyDeviceToHost = 2,
+ MemcpyDeviceToDevice = 3,
+ MemcpyDefault = 4
+};
+
+/// Malloc, with type template overlay.
+///{
+Error_t Malloc(void **Dev_Ptr, size_t Size);
+
+template <class T> static inline Error_t Malloc(T **dev_Ptr, size_t Size) {
+ return ::Malloc((void **)dev_Ptr, Size);
+}
+
+Error_t HostAlloc(void **Ptr, size_t Size, unsigned int Flags);
+
+template <class T>
+static inline Error_t HostAlloc(T **Ptr, size_t Size, unsigned int Flags) {
+ return ::HostAlloc((void **)Ptr, Size, Flags);
+}
+
+Error_t MallocHost(void **Ptr, size_t Size);
+
+template <class T> static inline Error_t MallocHost(T **Ptr, size_t Size) {
+ return ::MallocHost((void **)Ptr, Size);
+}
+///}
+
+/// Free, no type template necessary.
+Error_t Free(void *Dev_Ptr);
+
+/// Memcpy, with type template overlay.
+///{
+Error_t Memcpy(void *Dst, const void *Src, size_t Size, MemcpyKind Kind);
+
+template <class T>
+static inline Error_t Memcpy(T *Dst, const T *Src, size_t Size,
+ MemcpyKind Kind) {
+ return ::Memcpy((void *)Dst, (const void *)Src, Size, Kind);
+}
+///}
+
+/// DeviceSynchronize.
+Error_t DeviceSynchronize();
+
+Error_t GetLastError();
+
+Error_t PeekAtLastError();
+
+const char *GetErrorName(Error_t Error);
+
+const char *GetErrorString(Error_t Error);
+
+Error_t GetDeviceCount(int *Count);
+
+Error_t SetDevice(int DeviceNo);
+
+Error_t HostFree(void *Ptr);
+
+Error_t DriverGetVersion(int *Version);
+
+Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo);
+
+///
+
+#if defined(__AMDGPU__) || defined(__NVPTX__)
+#include <gpuintrin.h>
+
+#define __LLVM_OFFLOAD_DEVICE_BUILTIN(FIELD, OFFSET) \
+ __declspec( \
+ property(get = __get_##FIELD, put = __put_##FIELD)) unsigned int FIELD; \
+ __device__ inline __attribute__((always_inline)) T __get_##FIELD(void) \
+ const { \
+ return Vec[OFFSET]; \
+ } \
+ __device__ inline __attribute__((always_inline)) T __put_##FIELD(T V) { \
+ return Vec[OFFSET] = V; \
+ }
+
+template <class T, int Size> struct BaseVector {
+ using VT = float __attribute__((ext_vector_type(Size)));
+ VT Vec;
+
+ __device__ __host__ BaseVector() = default;
+// __device__ __host__ BaseVector(std::initializer_list<T> List) {
+// auto It = List.begin();
+// for (int I = 0, E = List.size(); I < E; ++I, ++It)
+// Vec[I] = *It;
+// }
+
+ template <typename... Args>
+ __device__ __host__ BaseVector(Args... args) : BaseVector({args...}) {}
+
+ __device__ __host__ T &operator[](int Idx) { return Vec[Idx]; }
+ __device__ __host__ const T &operator[](int Idx) const { return Vec[Idx]; }
+
+ __LLVM_OFFLOAD_DEVICE_BUILTIN(x, 0);
+ __LLVM_OFFLOAD_DEVICE_BUILTIN(y, 1);
+ __LLVM_OFFLOAD_DEVICE_BUILTIN(z, 2);
+ __LLVM_OFFLOAD_DEVICE_BUILTIN(w, 3);
+};
+
+#define __VECTOR_DEF_IMPL(TY, SIZE) \
+ using TY##SIZE = BaseVector<TY, SIZE>; \
+ \
+ template <typename... Args> \
+ __device__ __host__ TY##SIZE make_##TY##SIZE(Args... args) { \
+ return TY##SIZE(args...); \
+ }
+
+#define __VECTOR_DEF(TY) \
+ __VECTOR_DEF_IMPL(TY, 1) \
+ __VECTOR_DEF_IMPL(TY, 2) \
+ __VECTOR_DEF_IMPL(TY, 3) \
+ __VECTOR_DEF_IMPL(TY, 4) \
+ __VECTOR_DEF_IMPL(TY, 8) \
+ __VECTOR_DEF_IMPL(TY, 16)
+
+__VECTOR_DEF(float)
+__VECTOR_DEF(double)
+__VECTOR_DEF(int8_t)
+__VECTOR_DEF(int16_t)
+__VECTOR_DEF(int32_t)
+__VECTOR_DEF(int64_t)
+__VECTOR_DEF(uint8_t)
+__VECTOR_DEF(uint16_t)
+__VECTOR_DEF(uint32_t)
+__VECTOR_DEF(uint64_t)
+__VECTOR_DEF(char)
+__VECTOR_DEF(short)
+__VECTOR_DEF(int)
+__VECTOR_DEF(unsigned)
+__VECTOR_DEF(long)
+
+#undef __VECTOR_DEF_IMPL
+#undef __VECTOR_DEF
+#undef __LLVM_OFFLOAD_DEVICE_BUILTIN
+
+#endif
diff --git a/offload/languages/include/kernel/UndefineLanguageNames.inc b/offload/languages/include/kernel/UndefineLanguageNames.inc
new file mode 100644
index 0000000000000..e7cc890eaa2d1
--- /dev/null
+++ b/offload/languages/include/kernel/UndefineLanguageNames.inc
@@ -0,0 +1,34 @@
+//===-- LanguageNames.inc - Kernel Language runtime API names -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#undef Error_t
+#undef DeviceProp_t
+#undef Malloc
+#undef Free
+#undef Memcpy
+#undef DeviceSynchronize
+#undef Success
+#undef MemcpyKind
+#undef MemcpyHostToHost
+#undef MemcpyHostToDevice
+#undef MemcpyDeviceToHost
+#undef MemcpyDeviceToDevice
+#undef MemcpyDefault
+#undef GetLastError
+#undef PeekAtLastError
+#undef GetErrorName
+#undef GetErrorString
+#undef GetDeviceCount
+#undef SetDevice
+#undef HostAlloc
+#undef MallocHost
+#undef HostFree
+#undef DriverGetVersion
+#undef GetDeviceProperties
diff --git a/offload/languages/kernel/CMakeLists.txt b/offload/languages/kernel/CMakeLists.txt
new file mode 100644
index 0000000000000..22422ee601bb6
--- /dev/null
+++ b/offload/languages/kernel/CMakeLists.txt
@@ -0,0 +1,41 @@
+add_llvm_library(
+ LLVMOffloadKernel SHARED
+
+ ../kernel/src/State.cpp
+ ../kernel/src/ExportedAPI.cpp
+
+ LINK_COMPONENTS
+ Support
+ Offload
+ )
+
+if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+ target_link_libraries(LLVMOffloadKernel PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+endif()
+
+target_include_directories(LLVMOffloadKernel PUBLIC
+ ${CMAKE_CURRENT_BINARY_DIR}/../include
+ ${CMAKE_CURRENT_SOURCE_DIR}/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../kernel/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include/generated
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../plugins-nextgen/common/include)
+
+target_compile_options(LLVMOffloadKernel PRIVATE ${offload_compile_flags})
+target_link_options(LLVMOffloadKernel PRIVATE ${offload_link_flags})
+
+target_compile_definitions(LLVMOffloadKernel PRIVATE
+ TARGET_NAME="libLLVMOffloadKernel"
+ DEBUG_PREFIX="LLVMOffloadKernel"
+)
+
+set_target_properties(LLVMOffloadKernel PROPERTIES
+ POSITION_INDEPENDENT_CODE ON
+ INSTALL_RPATH "$ORIGIN"
+ BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
+install(TARGETS LLVMOffloadKernel LIBRARY COMPONENT LLVMOffloadKernel DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
+
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel/DefineLanguageNames.inc DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/kernel/)
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel/LanguageRuntime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/kernel/)
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel/UndefineLanguageNames.inc DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/kernel/)
diff --git a/offload/languages/kernel/exports b/offload/languages/kernel/exports
new file mode 100644
index 0000000000000..4bc5a4ab9710c
--- /dev/null
+++ b/offload/languages/kernel/exports
@@ -0,0 +1,8 @@
+VERS1.0 {
+ global:
+
+ olK*;
+
+ local:
+ *;
+};
diff --git a/offload/languages/kernel/include/ExportedAPI.h b/offload/languages/kernel/include/ExportedAPI.h
new file mode 100644
index 0000000000000..4f70c260c2b22
--- /dev/null
+++ b/offload/languages/kernel/include/ExportedAPI.h
@@ -0,0 +1,31 @@
+/*===---- ExportedAPI.h - Kernel language runtime - exported api ----------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#pragma once
+
+#include "OffloadAPI.h"
+#include "Types.h"
+
+extern "C" {
+ ol_device_handle_t olKGetDefaultDevice();
+
+ ol_device_handle_t olKGetHostDevice();
+
+ ol_queue_handle_t olKGetDefaultQueue();
+
+ CallConfigurationTy *olKGetCallConfiguration();
+
+ void olKRegisterKernel(const void *ID, ol_symbol_handle_t Kernel);
+
+ ol_symbol_handle_t olKGetKernel(const void *ID);
+
+ void olKRegisterProgram(const void *ID, ol_program_handle_t Program);
+
+ ol_program_handle_t olKGetProgram(const void *ID);
+}
diff --git a/offload/languages/kernel/include/LanguageAliases.h b/offload/languages/kernel/include/LanguageAliases.h
new file mode 100644
index 0000000000000..8a0d423b4cba7
--- /dev/null
+++ b/offload/languages/kernel/include/LanguageAliases.h
@@ -0,0 +1,40 @@
+//===------- Aliases.h --- Helpers to make symbol aliases -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#ifndef LANGUAGE
+#error This file should be included, or used, with a LANGUAGE macro set.
+#endif
+
+#define MA_IMPL2(PREFIX, L, RTY, NAME, ...) \
+ extern "C" [[gnu::alias("__llvm" #NAME)]] RTY PREFIX##L##NAME(__VA_ARGS__);
+
+#define MA_IMPL1(PREFIX, L, RTY, NAME, ...) \
+ MA_IMPL2(PREFIX, L, RTY, NAME, __VA_ARGS__)
+
+#define MAKE_ALIAS(PREFIX, RTY, NAME, ...) \
+ MA_IMPL1(PREFIX, LANGUAGE, RTY, NAME, __VA_ARGS__)
+
+MAKE_ALIAS(__, void, RegisterFunction, const char *, const char *, char *,
+ const char *, int, uint3 *, uint3 *, dim3 *, dim3 *, int *)
+MAKE_ALIAS(__, const char *, RegisterFatBinary, const char *)
+MAKE_ALIAS(__, void, UnregisterFatBinary, void *)
+MAKE_ALIAS(__, void, RegisterVar, void **, char *, char *, const char *, int,
+ int, int, int)
+MAKE_ALIAS(__, void, RegisterManagedVar, void **, char *, char *, const char *,
+ size_t, unsigned)
+MAKE_ALIAS(__, void, RegisterSurface, void **, const struct surfaceReference *,
+ const void **, const char *, int, int)
+MAKE_ALIAS(__, void, RegisterTexture, void **, const struct textureReference *,
+ const void **, const char *, int, int, int)
+
+MAKE_ALIAS(__, unsigned, PushCallConfiguration, dim3, dim3, size_t, void *)
+MAKE_ALIAS(__, unsigned, PopCallConfiguration, dim3 *, dim3 *, size_t *, void *)
diff --git a/offload/languages/kernel/include/LanguageLaunch.h b/offload/languages/kernel/include/LanguageLaunch.h
new file mode 100644
index 0000000000000..ece74815a1926
--- /dev/null
+++ b/offload/languages/kernel/include/LanguageLaunch.h
@@ -0,0 +1,51 @@
+//===------ LanguageLaunch.h - Header for LanguageLaunch.cpp ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LANGUAGE_LAUNCH_H
+#define LLVM_LANGUAGE_LAUNCH_H
+
+#include "ExportedAPI.h"
+#include "OffloadAPI.h"
+#include "Types.h"
+
+#include <algorithm> // for std::max
+#include <cstddef>
+#include <cstdint>
+
+extern "C" {
+
+/// Push call configuration for kernel launch
+unsigned __llvmPushCallConfiguration(dim3 __grid_size, dim3 __block_size,
+ size_t __shared_memory, void *__stream);
+
+/// Pop call configuration for kernel launch
+unsigned __llvmPopCallConfiguration(dim3 *__grid_size, dim3 *__block_size,
+ size_t *__shared_memory, void *__stream);
+
+/// Internal kernel launch implementation
+ol_result_t __llvmLaunchKernelImpl(const char *KernelID, dim3 GridDim,
+ dim3 BlockDim, void *KernelArgsPtr,
+ size_t DynamicSharedMem, void *Stream);
+
+/// LLVM-style kernel launch entry points
+unsigned __llvmLaunchKernel(const char *KernelID, dim3 GridDim, dim3 BlockDim,
+ void *KernelArgsPtr, size_t DynamicSharedMem,
+ void *Stream);
+
+unsigned __llvmLaunchKernel_spt(const char *KernelID, dim3 GridDim,
+ dim3 BlockDim, void *KernelArgsPtr,
+ size_t DynamicSharedMem, void *Stream);
+
+unsigned __llvmLaunchKernel_ptsz(const char *KernelID, dim3 GridDim,
+ dim3 BlockDim, void *KernelArgsPtr,
+ size_t DynamicSharedMem, void *Stream);
+}
+
+#endif // LLVM_LANGUAGE_LAUNCH_H
diff --git a/offload/languages/kernel/include/LanguageRegistration.h b/offload/languages/kernel/include/LanguageRegistration.h
new file mode 100644
index 0000000000000..307d738642ecf
--- /dev/null
+++ b/offload/languages/kernel/include/LanguageRegistration.h
@@ -0,0 +1,62 @@
+//===---- LanguageRegistration.h - Language (CUDA/HIP) registration api ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "ExportedAPI.h"
+
+#include "OffloadAPI.h"
+
+#include <cstdint>
+#include <iterator>
+
+#define HIP_FATBIN_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+constexpr auto HIP_FATBIN_MAGIC_STR_LEN = sizeof(HIP_FATBIN_MAGIC_STR) - 1;
+
+namespace {
+struct FatbinWrapperTy {
+ int Magic;
+ int Version;
+ const char *Data;
+ const char *DataEnd;
+};
+} // namespace
+
+static void readTUFatbin(const char *Binary, const FatbinWrapperTy *FW);
+
+static void readHIPFatbinEntries(const char *Binary, const char *HIPFatbinPtr);
+
+/// Hidden, but exported, Registration API
+///{
+extern "C" {
+
+void __llvmRegisterFunction(const char *Binary, const char *KernelID,
+ char *KernelName, const char *KernelName1, int,
+ uint3 *, uint3 *, dim3 *, dim3 *, int *);
+
+const char *__llvmRegisterFatBinary(const char *Binary);
+
+void __llvmUnregisterFatBinary(void *Handle);
+
+void __llvmRegisterVar(void **, char *, char *, const char *, int, int, int,
+ int);
+
+void __llvmRegisterManagedVar(void **, char *, char *, const char *, size_t,
+ unsigned);
+
+void __llvmRegisterSurface(void **, const struct surfaceReference *,
+ const void **, const char *, int, int);
+
+void __llvmRegisterTexture(void **, const struct textureReference *,
+ const void **, const char *, int, int, int);
+
+struct __tgt_bin_desc;
+void __tgt_register_lib(__tgt_bin_desc *Desc);
+void __tgt_unregister_lib(__tgt_bin_desc *Desc);
+}
+///}
diff --git a/offload/languages/kernel/include/Registration.h b/offload/languages/kernel/include/Registration.h
new file mode 100644
index 0000000000000..70f6a641f538d
--- /dev/null
+++ b/offload/languages/kernel/include/Registration.h
@@ -0,0 +1,17 @@
+//===-- Registration.h - Kernel Language (CUDA/HIP) registration handling -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+namespace llvm {
+namespace offload {
+void readHIPFatbin(const char *Binary, const char *HIPFatbinPtr);
+} // namespace offload
+} // namespace llvm
diff --git a/offload/languages/kernel/include/State.h b/offload/languages/kernel/include/State.h
new file mode 100644
index 0000000000000..02ad623b7b57d
--- /dev/null
+++ b/offload/languages/kernel/include/State.h
@@ -0,0 +1,92 @@
+//===------- State.h - Kernel Language (CUDA/HIP) persistent state --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "OffloadAPI.h"
+#include "Types.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+namespace offload {
+
+using KernelIDTy = const void *;
+
+struct ThreadStateTy {
+ ~ThreadStateTy();
+
+ static ThreadStateTy &get();
+
+ static ol_queue_handle_t getDefaultQueue();
+ static ol_device_handle_t getDefaultDevice();
+ static CallConfigurationTy &getCallConfiguration();
+
+private:
+ void setDefaultDevice(ol_device_handle_t Device);
+ void createDefaultQueue(ol_device_handle_t Device);
+
+ ol_device_handle_t DefaultDevice = nullptr;
+ ol_queue_handle_t DefaultQueue = nullptr;
+
+ CallConfigurationTy CC = {};
+
+ ThreadStateTy();
+};
+
+struct StateTy {
+ ~StateTy();
+
+ friend struct ThreadStateTy;
+
+ static StateTy &get();
+
+ static ol_device_handle_t getHostDevice() { return get().HostDevice; }
+
+ ArrayRef<ol_device_handle_t> getDevices() const { return Devices; }
+
+ void addDevice(ol_device_handle_t Device) { Devices.push_back(Device); }
+ void setHostDevice(ol_device_handle_t Device) {
+ if (!HostDevice)
+ HostDevice = Device;
+ }
+
+ void addKernel(KernelIDTy KernelID, ol_symbol_handle_t Kernel) {
+ KernelMap[KernelID] = Kernel;
+ }
+
+ ol_symbol_handle_t getKernel(KernelIDTy KernelID) {
+ return KernelMap[KernelID];
+ }
+
+ void addProgram(const void *Binary, ol_program_handle_t Program) {
+ BinaryRegisterMap[Binary] = Program;
+ }
+
+ ol_program_handle_t getProgram(const void *Binary) {
+ assert(BinaryRegisterMap.count(Binary));
+ return BinaryRegisterMap[Binary];
+ }
+
+private:
+ DenseMap<const void *, ol_program_handle_t> BinaryRegisterMap;
+ DenseMap<KernelIDTy, ol_symbol_handle_t> KernelMap;
+ SmallVector<ol_device_handle_t, 8> Devices;
+
+ ol_queue_handle_t DefaultQueue = nullptr;
+ ol_device_handle_t HostDevice = nullptr;
+
+ StateTy();
+};
+
+} // namespace offload
+} // namespace llvm
diff --git a/offload/languages/kernel/include/Types.h b/offload/languages/kernel/include/Types.h
new file mode 100644
index 0000000000000..6c5a0ec19a8f7
--- /dev/null
+++ b/offload/languages/kernel/include/Types.h
@@ -0,0 +1,28 @@
+//===------- Types.h - Kernel Language (CUDA/HIP) api types ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "Types.h"
+#include <cstddef>
+#include <cstdint>
+
+struct uint3 {
+ unsigned x = 0, y = 0, z = 0;
+};
+
+using dim3 = uint3;
+
+struct CallConfigurationTy {
+ dim3 GridSize;
+ dim3 BlockSize;
+ size_t SharedMemory;
+ void *Stream;
+};
diff --git a/offload/languages/kernel/src/ExportedAPI.cpp b/offload/languages/kernel/src/ExportedAPI.cpp
new file mode 100644
index 0000000000000..d3f8847744fbb
--- /dev/null
+++ b/offload/languages/kernel/src/ExportedAPI.cpp
@@ -0,0 +1,60 @@
+//===------ ExportedAPI.cpp - Kernel Language runtime - exported api ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "ExportedAPI.h"
+
+#include "State.h"
+#include "Types.h"
+
+#include "OffloadAPI.h"
+
+#include <cstdio>
+#include <stdint.h>
+
+using namespace llvm;
+using namespace offload;
+
+/// Runtime API
+///{
+ol_device_handle_t olKGetDefaultDevice() {
+ ol_device_handle_t DefaultDevice = ThreadStateTy::getDefaultDevice();
+ return DefaultDevice;
+}
+
+ol_device_handle_t olKGetHostDevice() {
+ ol_device_handle_t HostDevice = StateTy::getHostDevice();
+ return HostDevice;
+}
+
+ol_queue_handle_t olKGetDefaultQueue() {
+ ol_queue_handle_t DefaultQueue = ThreadStateTy::getDefaultQueue();
+ return DefaultQueue;
+}
+
+CallConfigurationTy* olKGetCallConfiguration() {
+ return &ThreadStateTy::getCallConfiguration();
+}
+
+void olKRegisterKernel(const void *ID, ol_symbol_handle_t Kernel) {
+ StateTy::get().addKernel(ID, Kernel);
+}
+
+ol_symbol_handle_t olKGetKernel(const void *ID) {
+ return StateTy::get().getKernel(ID);
+}
+
+void olKRegisterProgram(const void *ID, ol_program_handle_t Program) {
+ StateTy::get().addProgram(ID, Program);
+}
+
+ol_program_handle_t olKGetProgram(const void *ID) {
+ return StateTy::get().getProgram(ID);
+}
+///}
diff --git a/offload/languages/kernel/src/LanguageLaunch.cpp b/offload/languages/kernel/src/LanguageLaunch.cpp
new file mode 100644
index 0000000000000..685c69607c67c
--- /dev/null
+++ b/offload/languages/kernel/src/LanguageLaunch.cpp
@@ -0,0 +1,94 @@
+//===------ LanguageLaunch.cpp - Language (CUDA/HIP) launch api -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "LanguageLaunch.h"
+
+#include <cstdio>
+
+extern "C" {
+
+/// Push call configuration for kernel launch
+unsigned __llvmPushCallConfiguration(dim3 __grid_size, dim3 __block_size,
+ size_t __shared_memory, void *__stream) {
+ CallConfigurationTy &CC = *olKGetCallConfiguration();
+
+ CC.GridSize = __grid_size;
+ CC.BlockSize = __block_size;
+ CC.SharedMemory = __shared_memory;
+ CC.Stream = __stream;
+ return 0;
+}
+
+/// Pop call configuration for kernel launch
+unsigned __llvmPopCallConfiguration(dim3 *__grid_size, dim3 *__block_size,
+ size_t *__shared_memory, void *__stream) {
+ CallConfigurationTy &CC = *olKGetCallConfiguration();
+ *__grid_size = CC.GridSize;
+ *__block_size = CC.BlockSize;
+ *__shared_memory = CC.SharedMemory;
+ *((void **)__stream) = CC.Stream;
+ return 0;
+}
+
+/// Internal kernel launch implementation
+ol_result_t __llvmLaunchKernelImpl(const char *KernelID, dim3 GridDim,
+ dim3 BlockDim, void *KernelArgsPtr,
+ size_t DynamicSharedMem, void *Stream) {
+ ol_device_handle_t Device = olKGetDefaultDevice();
+ ol_symbol_handle_t Kernel = olKGetKernel(KernelID);
+
+ ol_dimensions_t GridDimensions, BlockDimensions;
+ ol_kernel_launch_size_args_t LaunchSizeArgs;
+ LaunchSizeArgs.Dimensions =
+ 1 + !!(GridDim.y * BlockDim.y > 1) + !!(GridDim.z * BlockDim.z > 1);
+ GridDimensions.x = GridDim.x;
+ GridDimensions.y = std::max(GridDim.y, 1u);
+ GridDimensions.z = std::max(GridDim.z, 1u);
+ LaunchSizeArgs.NumGroups = GridDimensions;
+ BlockDimensions.x = BlockDim.x;
+ BlockDimensions.y = std::max(BlockDim.y, 1u);
+ BlockDimensions.z = std::max(BlockDim.z, 1u);
+ LaunchSizeArgs.GroupSize = BlockDimensions;
+ LaunchSizeArgs.DynSharedMemory = DynamicSharedMem;
+
+ ol_queue_handle_t Queue = Stream ? reinterpret_cast<ol_queue_handle_t>(Stream)
+ : olKGetDefaultQueue();
+
+ ol_kernel_launch_prop_t Properties = {.type = OL_KERNEL_LAUNCH_PROP_TYPE_NONE,
+ .data = nullptr};
+
+ struct OffloadKernelArgs {
+ void **Args;
+ size_t NumArgs;
+ size_t *ArgSizes;
+ };
+ OffloadKernelArgs *OKA = reinterpret_cast<OffloadKernelArgs*>(KernelArgsPtr);
+
+ ol_result_t Result;
+ Result = olLaunchKernel(Queue, Device, Kernel, &LaunchSizeArgs, &Properties,
+ OKA->NumArgs, OKA->Args, OKA->ArgSizes);
+ return Result;
+}
+
+#define LLVM_STYLE_LAUNCH(SUFFIX, PER_THREAD_STREAM) \
+ unsigned __llvmLaunchKernel##SUFFIX(const char *KernelID, dim3 GridDim, \
+ dim3 BlockDim, void *KernelArgsPtr, \
+ size_t DynamicSharedMem, void *Stream) { \
+ ol_result_t Result = \
+ __llvmLaunchKernelImpl(KernelID, GridDim, BlockDim, KernelArgsPtr, \
+ DynamicSharedMem, Stream); \
+ return Result ? Result->Code : 0; \
+ }
+
+LLVM_STYLE_LAUNCH(, false);
+LLVM_STYLE_LAUNCH(_spt, true);
+LLVM_STYLE_LAUNCH(_ptsz, true);
+
+} // extern "C"
diff --git a/offload/languages/kernel/src/LanguageRegistration.cpp b/offload/languages/kernel/src/LanguageRegistration.cpp
new file mode 100644
index 0000000000000..d0c7cae5d007a
--- /dev/null
+++ b/offload/languages/kernel/src/LanguageRegistration.cpp
@@ -0,0 +1,311 @@
+//===---- LanguageRegistration.h - Language (CUDA/HIP) registration api ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "LanguageRegistration.h"
+#include "OffloadAPI.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/Offloading/Utility.h"
+#include "llvm/Support/Error.h"
+#include <inttypes.h>
+
+typedef struct __attribute__((__packed__)) {
+ uint32_t Magic;
+ uint16_t Version;
+ uint16_t HeaderSize;
+ uint64_t FatSize;
+} CudaFatbinHeader;
+
+// Inspired by
+// https://github.com/n-eiling/cuda-fatbin-decompression/blob/master/fatbin-decompress.h
+typedef struct __attribute__((__packed__)) {
+ uint16_t Kind;
+ uint16_t Unknown1;
+ uint32_t HeaderSize;
+ uint64_t Size;
+ uint32_t CompressedSize;
+ uint32_t Unknown2;
+ uint16_t Minor;
+ uint16_t Major;
+ uint32_t Arch;
+ uint32_t ObjNameOffset;
+ uint32_t ObjNameLen;
+ uint64_t Flags;
+ uint64_t Zero;
+ uint64_t DecompressedSize;
+} CudaFatbinTextHeader;
+
+// HIP uses this format:
+// https://clang.llvm.org/docs/ClangOffloadBundler.html#bundled-binary-file-layout
+typedef struct __attribute__((__packed__)) {
+ char Magic[24];
+ uint64_t NumBundles;
+} HipFatbinHeader;
+
+typedef struct __attribute__((__packed__)) {
+ uint64_t BundleOffset;
+ uint64_t BundleSize;
+ uint64_t IdLength;
+ char IdString[];
+} HipFatbinBundleEntry;
+
+static void readTUFatbin(const char *Binary, const FatbinWrapperTy *FW) {
+ ol_device_handle_t Device = olKGetDefaultDevice();
+
+ const CudaFatbinHeader *Header =
+ reinterpret_cast<const CudaFatbinHeader *>(FW->Data);
+ size_t HeaderSize = static_cast<size_t>(Header->HeaderSize); // Usually 16
+ size_t FatbinSize = static_cast<size_t>(Header->FatSize);
+
+ const void *ProgramData = nullptr;
+ size_t ProgramSize = 0;
+ uint32_t ProgramArch = 0;
+
+ const char *ReadPosition = FW->Data + HeaderSize;
+ while (ReadPosition < (FW->Data + FatbinSize)) {
+ const CudaFatbinTextHeader *TextHeader =
+ reinterpret_cast<const CudaFatbinTextHeader *>(ReadPosition);
+ size_t TextHeaderSize =
+ static_cast<size_t>(TextHeader->HeaderSize); // Usually 64
+ size_t CubinSize = static_cast<size_t>(TextHeader->Size);
+ const void *CubinData =
+ static_cast<const char *>(ReadPosition + TextHeaderSize);
+
+ uint32_t Arch = TextHeader->Arch;
+ bool IsCompatible = false;
+ ol_result_t CompatibilityCheckResult = olElfIsCompatibleWithDevice(
+ Device, CubinData, CubinSize, &IsCompatible);
+
+ if (CompatibilityCheckResult && CompatibilityCheckResult->Code) {
+ fprintf(stderr, "Failed to check for device compatibility (%i): %s\n",
+ CompatibilityCheckResult->Code,
+ CompatibilityCheckResult->Details);
+ abort();
+ }
+
+ if (IsCompatible && Arch > ProgramArch) {
+ ProgramData = CubinData;
+ ProgramSize = CubinSize;
+ ProgramArch = Arch;
+ }
+
+ ReadPosition += TextHeaderSize + CubinSize;
+ }
+
+ if (ProgramData == nullptr) {
+ fprintf(stderr, "Failed to find compatible binary\n");
+ abort();
+ }
+
+ ol_program_handle_t Program = nullptr;
+
+ ol_result_t Result =
+ olCreateProgram(Device, ProgramData, ProgramSize, &Program);
+
+ if (Result && Result->Code) {
+ fprintf(stderr, "Failed to register device code (%i): %s\n", Result->Code,
+ Result->Details);
+ abort();
+ }
+
+ olKRegisterProgram(Binary, Program);
+}
+
+static void readHIPFatbinEntries(const char *Binary, const char *HIPFatbinPtr) {
+ ol_device_handle_t Device = olKGetDefaultDevice();
+
+ const char *CurrentReadPosition = HIPFatbinPtr;
+
+ const HipFatbinHeader *Header =
+ reinterpret_cast<const HipFatbinHeader *>(CurrentReadPosition);
+ CurrentReadPosition += sizeof(HipFatbinHeader);
+
+ uint64_t NumBundles = Header->NumBundles;
+
+ const void *ProgramData = nullptr;
+ size_t ProgramSize = 0;
+ uint64_t ProgramIdLength = 0;
+ const char *ProgramIdString = nullptr;
+
+ for (uint64_t BundleId = 0; BundleId < NumBundles; ++BundleId) {
+ const HipFatbinBundleEntry *BundleEntry =
+ reinterpret_cast<const HipFatbinBundleEntry *>(CurrentReadPosition);
+
+ uint64_t BundleOffset = BundleEntry->BundleOffset;
+ uint64_t BundleSize = BundleEntry->BundleSize;
+ const char *BundleIdString = BundleEntry->IdString;
+ uint64_t BundleIdLength = BundleEntry->IdLength;
+
+ // Advance by the size of the entry including the ID string
+ CurrentReadPosition += sizeof(HipFatbinBundleEntry) + BundleIdLength;
+
+ if (!BundleSize) {
+ continue;
+ }
+
+ bool IsCompatible = false;
+ ol_result_t CompatibilityCheckResult = olElfIsCompatibleWithDevice(
+ Device, HIPFatbinPtr + BundleOffset, BundleSize, &IsCompatible);
+
+ if (CompatibilityCheckResult && CompatibilityCheckResult->Code) {
+ fprintf(stderr, "Failed to check for device compatibility (%i): %s\n",
+ CompatibilityCheckResult->Code,
+ CompatibilityCheckResult->Details);
+ abort();
+ }
+
+ llvm::StringRef CurrentBundleId(ProgramIdString, ProgramIdLength);
+ llvm::StringRef NewBundleId(BundleIdString, BundleIdLength);
+ if (IsCompatible && NewBundleId.compare(CurrentBundleId) > 0) {
+ ProgramData = HIPFatbinPtr + BundleOffset;
+ ProgramSize = BundleSize;
+ ProgramIdLength = BundleIdLength;
+ ProgramIdString = BundleIdString;
+ }
+ }
+
+ if (ProgramData == nullptr) {
+ fprintf(stderr, "Failed to find compatible binary\n");
+ abort();
+ }
+
+ ol_program_handle_t Program = nullptr;
+ ol_result_t Result =
+ olCreateProgram(Device, ProgramData, ProgramSize, &Program);
+ if (Result && Result->Code) {
+ fprintf(stderr, "Failed to register device code (%i): %s\n", Result->Code,
+ Result->Details);
+ abort();
+ }
+
+ olKRegisterProgram(Binary, Program);
+}
+
+/// Hidden, but exported, Registration API
+///{
+extern "C" {
+
+void __llvmRegisterFunction(const char *Binary, const char *KernelID,
+ char *KernelName, const char *KernelName1, int,
+ uint3 *, uint3 *, dim3 *, dim3 *, int *) {
+ printf("%s :: %p :: %p : %s : %s \n", __PRETTY_FUNCTION__, Binary, KernelID,
+ KernelName, KernelName1);
+ ol_symbol_handle_t Kernel;
+ ol_program_handle_t Program = olKGetProgram(Binary);
+ ol_result_t Result = olGetSymbol(
+ Program, KernelName, ol_symbol_kind_t::OL_SYMBOL_KIND_KERNEL, &Kernel);
+ if (Result && Result->Code) {
+ fprintf(stderr, "Failed to register kernel (%i): %s\n", Result->Code,
+ Result->Details);
+ abort();
+ }
+
+ printf("K %p : %p\n", KernelID, Kernel);
+ olKRegisterKernel(KernelID, Kernel);
+}
+
+const char *__llvmRegisterFatBinary(const char *Binary) {
+
+ const auto *FW = reinterpret_cast<const FatbinWrapperTy *>(Binary);
+ // printf("%s : %i : %s (%p:%p) :: %i\n", __PRETTY_FUNCTION__, FW->Magic,
+ // FW->Data, FW->Data, FW->DataEnd, FW->Version);
+
+ // printf("%s : %s : %lu\n", FW->Data, HIP_FATBIN_MAGIC_STR,
+ // HIP_FATBIN_MAGIC_STR_LEN);
+ if (FW->Magic == 0x466243b1) {
+ readTUFatbin(Binary, FW);
+ } else if (FW->Magic == 0x48495046) {
+ if (!memcmp(FW->Data, HIP_FATBIN_MAGIC_STR, HIP_FATBIN_MAGIC_STR_LEN))
+ readHIPFatbinEntries(Binary, FW->Data);
+ else
+ readTUFatbin(Binary, FW);
+ } else {
+ fprintf(stderr, "Unknown fatbin format");
+ }
+
+ return Binary;
+}
+
+void __llvmUnregisterFatBinary(void *Handle) {}
+
+void __llvmRegisterVar(void **, char *, char *, const char *, int, int, int,
+ int) {
+ fprintf(stderr, "RegisterVar is not implemented!");
+}
+
+void __llvmRegisterManagedVar(void **, char *, char *, const char *, size_t,
+ unsigned) {
+ fprintf(stderr, "RegisterManagedVar is not implemented!");
+}
+
+void __llvmRegisterSurface(void **, const struct surfaceReference *,
+ const void **, const char *, int, int) {
+ fprintf(stderr, "RegisterSurface is not implemented!");
+}
+
+void __llvmRegisterTexture(void **, const struct textureReference *,
+ const void **, const char *, int, int, int) {
+ fprintf(stderr, "RegisterTexture is not implemented!");
+}
+
+/// This struct is a record of the device image information
+struct __tgt_device_image {
+ void *ImageStart; // Pointer to the target code start
+ void *ImageEnd; // Pointer to the target code end
+ llvm::offloading::EntryTy
+ *EntriesBegin; // Begin of table with all target entries
+ llvm::offloading::EntryTy *EntriesEnd; // End of table (non inclusive)
+};
+
+/// This struct is a record of all the host code that may be offloaded to a
+/// target.
+struct __tgt_bin_desc {
+ int32_t NumDeviceImages; // Number of device types supported
+ __tgt_device_image *DeviceImages; // Array of device images (1 per dev. type)
+ llvm::offloading::EntryTy
+ *HostEntriesBegin; // Begin of table with all host entries
+ llvm::offloading::EntryTy *HostEntriesEnd; // End of table (non inclusive)
+};
+
+void __tgt_register_lib(__tgt_bin_desc *Desc) {
+ // TODO: For each device, lazily.
+ ol_device_handle_t Device = olKGetDefaultDevice();
+
+ for (int32_t I = 0, E = Desc->NumDeviceImages; I < E; ++I) {
+ ol_program_handle_t Program = nullptr;
+
+ __tgt_device_image &DeviceImage = Desc->DeviceImages[I];
+ void *ProgramData = DeviceImage.ImageStart;
+ size_t ProgramSize =
+ (char *)DeviceImage.ImageEnd - (char *)DeviceImage.ImageStart;
+ ol_result_t Result =
+ olCreateProgram(Device, ProgramData, ProgramSize, &Program);
+
+ if (Result && Result->Code) {
+ fprintf(stderr, "Failed to register device code (%i): %s\n", Result->Code,
+ Result->Details);
+ abort();
+ }
+
+ olKRegisterProgram(DeviceImage.ImageStart, Program);
+
+ for (auto *Entry = DeviceImage.EntriesBegin;
+ Entry != DeviceImage.EntriesEnd; ++Entry) {
+ if (!Entry->Size && !Entry->Flags)
+ __llvmRegisterFunction((const char *)DeviceImage.ImageStart,
+ (const char*)Entry->Address, Entry->SymbolName,
+ Entry->SymbolName, 0, nullptr, nullptr, nullptr,
+ nullptr, nullptr);
+ }
+ }
+}
+
+void __tgt_unregister_lib(__tgt_bin_desc *Desc) {}
+}
+///}
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
new file mode 100644
index 0000000000000..3a08d33e977ec
--- /dev/null
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -0,0 +1,152 @@
+//===-- LanguageRuntime.cpp - Kernel Language runtime API implementation --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "LanguageRuntime.h"
+
+#ifndef LANGUAGE
+#error This file should be included, or used, with a LANGUAGE macro set.
+#endif
+
+#include "ExportedAPI.h"
+#include "Types.h"
+
+#include "OffloadAPI.h"
+
+#include "DefineLanguageNames.inc"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#define STR(X) #X
+#define LANGUAGE_STR STR(LANGUAGE)
+
+Error_t olKConvertResult(ol_result_t Result) { return Success; }
+
+Error_t Malloc(void **DevPtr, size_t Size) {
+ ol_device_handle_t Device = olKGetDefaultDevice();
+ ol_result_t Result = olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, DevPtr);
+ return olKConvertResult(Result);
+}
+
+Error_t Free(void *DevPtr) {
+ ol_result_t Result = olMemFree(DevPtr);
+ return olKConvertResult(Result);
+}
+
+Error_t Memcpy(void *Dst, const void *Src, size_t Size, MemcpyKind Kind) {
+ ol_queue_handle_t Queue = olKGetDefaultQueue();
+
+ ol_result_t Result;
+ switch (Kind) {
+ case MemcpyHostToHost: {
+ ol_device_handle_t Host = olKGetHostDevice();
+ Result = olMemcpy(Queue, Dst, Host, const_cast<void *>(Src), Host, Size);
+ break;
+ }
+ case MemcpyHostToDevice: {
+ ol_device_handle_t Device = olKGetDefaultDevice();
+ ol_device_handle_t Host = olKGetHostDevice();
+ Result = olMemcpy(Queue, Dst, Device, const_cast<void *>(Src), Host, Size);
+ break;
+ }
+ case MemcpyDeviceToHost: {
+ ol_device_handle_t Device = olKGetDefaultDevice();
+ ol_device_handle_t Host = olKGetHostDevice();
+
+ Result = olMemcpy(Queue, Dst, Host, const_cast<void *>(Src), Device, Size);
+ break;
+ }
+ case MemcpyDeviceToDevice: {
+ ol_device_handle_t Device = olKGetDefaultDevice();
+
+ Result =
+ olMemcpy(Queue, Dst, Device, const_cast<void *>(Src), Device, Size);
+ break;
+ }
+ case MemcpyDefault:
+ fprintf(stderr, LANGUAGE_STR "MemcpyDefault is not implemented yet");
+ abort();
+ };
+
+ Result = olSyncQueue(Queue);
+
+ return olKConvertResult(Result);
+}
+
+Error_t DeviceSynchronize() {
+ // TODO: This is not correct. We likely want to pipe this through to the
+ // plugins.
+ ol_queue_handle_t Queue = olKGetDefaultQueue();
+ ol_result_t Result = olSyncQueue(Queue);
+ return olKConvertResult(Result);
+}
+
+Error_t GetLastError() {
+ // TODO:
+ return Success;
+}
+
+Error_t PeekAtLastError() {
+ // TODO:
+ return Success;
+}
+
+const char *GetErrorName(Error_t Error) {
+ // TODO:
+ return "";
+}
+
+const char *GetErrorString(Error_t Error) {
+ // TODO:
+ return "";
+}
+
+Error_t GetDeviceCount(int *Count) {
+ // TODO:
+ *Count = 1;
+ return Success;
+}
+
+Error_t SetDevice(int DeviceNo) {
+ // TODO:
+ return Success;
+}
+
+Error_t HostAlloc(void **Ptr, size_t Size, unsigned int Flags) {
+ // TODO:
+ ol_device_handle_t Device = olKGetDefaultDevice();
+ ol_result_t Result = olMemAlloc(Device, OL_ALLOC_TYPE_HOST, Size, Ptr);
+ return olKConvertResult(Result);
+}
+
+Error_t MallocHost(void **Ptr, size_t Size) {
+ return HostAlloc(Ptr, Size, /* HostAllocDefault */ 0);
+}
+
+Error_t HostFree(void *Ptr) {
+ ol_result_t Result = olMemFree(Ptr);
+ return olKConvertResult(Result);
+}
+
+Error_t DriverGetVersion(int *Version) {
+ // TODO:
+ *Version = 42;
+ return Success;
+}
+
+Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
+ memcpy(&DeviceProp->name[0], "TESTGPU", sizeof("TESTGPU"));
+ DeviceProp->totalGlobalMem = 1024l * 1024 * 1024 * 40;
+ DeviceProp->multiProcessorCount = 110;
+ DeviceProp->major = 47;
+ DeviceProp->minor = 11;
+ return Success;
+}
diff --git a/offload/languages/kernel/src/State.cpp b/offload/languages/kernel/src/State.cpp
new file mode 100644
index 0000000000000..e6ef90a44c7de
--- /dev/null
+++ b/offload/languages/kernel/src/State.cpp
@@ -0,0 +1,167 @@
+//===------ State.cpp - Kernel Language (CUDA/HIP) persistent state -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "State.h"
+#include "Types.h"
+
+#include "OffloadAPI.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <atomic>
+#include <cstdio>
+#include <mutex>
+
+#define CHECK_FATAL(Result, ...) \
+ if (Result && Result->Code) { \
+ fprintf(stderr, __VA_ARGS__); \
+ abort(); \
+ }
+
+using namespace llvm;
+using namespace offload;
+
+std::atomic<uint32_t> AnyNonDefaultDevice = 0;
+__attribute__((weak)) uint32_t PerThreadQueue = 0;
+
+thread_local CallConfigurationTy CC = {};
+
+static std::mutex StateLock;
+static std::atomic<StateTy *> StatePtr = nullptr;
+
+static thread_local ThreadStateTy *ThreadState = nullptr;
+
+static std::mutex ThreadStatesLock;
+using ThreadStatesTy = SmallVector<ThreadStateTy *, 64>;
+static ThreadStatesTy *ThreadStatesPtr = nullptr;
+
+static void deleteThreadState() {
+ std::lock_guard<std::mutex> LG(ThreadStatesLock);
+ if (ThreadStatesPtr)
+ for (auto *TS : *ThreadStatesPtr)
+ delete (TS);
+}
+
+static void deleteState() {
+ StateTy *ST = StatePtr.load();
+ StatePtr.store(nullptr);
+ delete (ST);
+}
+
+ThreadStateTy::ThreadStateTy() {
+ if (PerThreadQueue) [[unlikely]]
+ createDefaultQueue(getDefaultDevice());
+ atexit(deleteThreadState);
+}
+ThreadStateTy::~ThreadStateTy() {
+ if (DefaultQueue)
+ olSyncQueue(DefaultQueue);
+}
+
+ThreadStateTy &ThreadStateTy::get() {
+ auto *TS = ThreadState;
+ if (!TS) {
+ TS = new ThreadStateTy();
+ ThreadState = TS;
+ std::lock_guard<std::mutex> LG(ThreadStatesLock);
+ if (!ThreadStatesPtr)
+ ThreadStatesPtr = new ThreadStatesTy;
+ ThreadStatesPtr->push_back(TS);
+ }
+ return *TS;
+}
+
+ol_device_handle_t ThreadStateTy::getDefaultDevice() {
+ ol_device_handle_t DD = nullptr;
+ for (ol_device_handle_t Device : StateTy::get().getDevices()) {
+ DD = Device;
+ break;
+ }
+ if (AnyNonDefaultDevice.load(std::memory_order_relaxed)) [[unlikely]] {
+ ol_device_handle_t TDD = ThreadStateTy::get().DefaultDevice;
+ if (TDD)
+ DD = TDD;
+ }
+ return DD;
+}
+
+ol_queue_handle_t ThreadStateTy::getDefaultQueue() {
+ if (!PerThreadQueue) [[likely]]
+ return StateTy::get().DefaultQueue;
+ return ThreadStateTy::get().DefaultQueue;
+}
+
+CallConfigurationTy& ThreadStateTy::getCallConfiguration() {
+ return ThreadStateTy::get().CC;
+}
+
+void ThreadStateTy::setDefaultDevice(ol_device_handle_t Device) {
+ DefaultDevice = Device;
+ createDefaultQueue(Device);
+}
+
+void ThreadStateTy::createDefaultQueue(ol_device_handle_t Device) {
+ if (DefaultQueue)
+ olDestroyQueue(DefaultQueue);
+ CHECK_FATAL(olCreateQueue(Device, &DefaultQueue),
+ "Failed to create per-thread default queue");
+}
+
+StateTy &StateTy::get() {
+ StateTy *ST = StatePtr.load();
+ if (!ST) [[unlikely]] {
+ std::lock_guard<std::mutex> LG(StateLock);
+ ST = StatePtr.load();
+ if (!ST) {
+ ST = new StateTy();
+ StatePtr.store(ST);
+ }
+ }
+ return *ST;
+}
+
+static bool addDevices(ol_device_handle_t Device, void *Payload) {
+ StateTy &State = *reinterpret_cast<StateTy *>(Payload);
+ ol_platform_handle_t Platform;
+ ol_result_t Result;
+
+ Result = olGetDeviceInfo(Device, OL_DEVICE_INFO_PLATFORM, sizeof(Platform),
+ &Platform);
+ if (Result && Result->Code)
+ return true;
+
+ ol_platform_backend_t Backend;
+ Result = olGetPlatformInfo(Platform, OL_PLATFORM_INFO_BACKEND,
+ sizeof(Backend), &Backend);
+ if (Result && Result->Code)
+ return true;
+
+ if (Backend == OL_PLATFORM_BACKEND_HOST)
+ State.setHostDevice(Device);
+ else
+ State.addDevice(Device);
+ return true;
+}
+
+StateTy::StateTy() {
+ CHECK_FATAL(olInit(nullptr), "Failed to initialize the LLVMOffload");
+ CHECK_FATAL(olIterateDevices(addDevices, this), "Failed to identify devices");
+
+ if (!PerThreadQueue) [[likely]]
+ if (!Devices.empty()) [[likely]]
+ CHECK_FATAL(olCreateQueue(Devices.front(), &DefaultQueue),
+ "Failed to create default queue");
+
+ atexit(deleteState);
+}
+
+StateTy::~StateTy() {
+ if (DefaultQueue)
+ olSyncQueue(DefaultQueue);
+}
diff --git a/offload/liboffload/API/Device.td b/offload/liboffload/API/Device.td
index 42052af7bd065..2f365bc9d6a1b 100644
--- a/offload/liboffload/API/Device.td
+++ b/offload/liboffload/API/Device.td
@@ -142,3 +142,27 @@ def olGetDeviceInfoSize : Function {
Return<"OL_ERRC_INVALID_DEVICE">
];
}
+
+def olElfIsCompatibleWithDevice : Function {
+ let desc = "Checks if the given ELF binary is compatible with the specified device.";
+ let details = [
+ "This function determines whether an ELF image can be executed on the specified device."
+ ];
+ let params = [
+ Param<"ol_device_handle_t", "Device", "handle of the device to check against", PARAM_IN>,
+ Param<"const void*", "ElfData", "pointer to the ELF image data in memory", PARAM_IN>,
+ Param<"size_t", "ElfSize", "size in bytes of the ELF image", PARAM_IN>,
+ Param<"bool*", "IsCompatible", "set to true if the ELF is compatible, false otherwise", PARAM_OUT>
+ ];
+ let returns = [
+ Return<"OL_ERRC_INVALID_DEVICE", [
+ "If the provided device handle is invalid."
+ ]>,
+ Return<"OL_ERRC_INVALID_ARGUMENT", [
+ "If `ElfData` is null or `ElfSize` is zero."
+ ]>,
+ Return<"OL_ERRC_NULL_POINTER", [
+ "If `IsCompatible` is null."
+ ]>
+ ];
+}
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index a36081f27b5ee..34cd0fe170115 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -584,6 +584,25 @@ Error olIterateDevices_impl(ol_device_iterate_cb_t Callback, void *UserData) {
return Error::success();
}
+Error olElfIsCompatibleWithDevice_impl(ol_device_handle_t Device,
+ const void *ElfData,
+ size_t ElfSize,
+ bool *IsCompatible) {
+ GenericDeviceTy *DeviceTy = Device->Device;
+ int32_t DeviceId = DeviceTy->getDeviceId();
+ GenericPluginTy &DevicePlugin = DeviceTy->Plugin;
+
+ StringRef Image(reinterpret_cast<const char *>(ElfData), ElfSize);
+
+ Expected<bool> ResultOrErr = DevicePlugin.isELFCompatible(DeviceId, Image);
+ if (!ResultOrErr)
+ return ResultOrErr.takeError();
+
+ *IsCompatible = *ResultOrErr;
+ return Error::success();
+}
+
+
TargetAllocTy convertOlToPluginAllocTy(ol_alloc_type_t Type) {
switch (Type) {
case OL_ALLOC_TYPE_DEVICE:
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 8e6314b2a6eae..29eaf2d24680c 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,8 +17,6 @@ add_library(omptarget SHARED
OpenMP/Mapping.cpp
OpenMP/InteropAPI.cpp
OpenMP/OMPT/Callback.cpp
-
- KernelLanguage/API.cpp
)
if(LLVM_LINK_LLVM_DYLIB)
diff --git a/offload/libomptarget/KernelLanguage/API.cpp b/offload/libomptarget/KernelLanguage/API.cpp
deleted file mode 100644
index 50f9b695bed6a..0000000000000
--- a/offload/libomptarget/KernelLanguage/API.cpp
+++ /dev/null
@@ -1,74 +0,0 @@
-//===------ API.cpp - Kernel Language (CUDA/HIP) entry points ----- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-
-#include "Shared/APITypes.h"
-
-#include <cstdio>
-
-struct dim3 {
- unsigned x = 0, y = 0, z = 0;
-};
-
-struct __omp_kernel_t {
- dim3 __grid_size;
- dim3 __block_size;
- size_t __shared_memory;
-
- void *__stream;
-};
-
-static __omp_kernel_t __current_kernel = {};
-#pragma omp threadprivate(__current_kernel);
-
-extern "C" {
-
-// TODO: There is little reason we need to keep these names or the way calls are
-// issued. For now we do to avoid modifying Clang's CUDA codegen. Unclear when
-// we actually need to push/pop configurations.
-unsigned __llvmPushCallConfiguration(dim3 __grid_size, dim3 __block_size,
- size_t __shared_memory, void *__stream) {
- __omp_kernel_t &__kernel = __current_kernel;
- __kernel.__grid_size = __grid_size;
- __kernel.__block_size = __block_size;
- __kernel.__shared_memory = __shared_memory;
- __kernel.__stream = __stream;
- return 0;
-}
-
-unsigned __llvmPopCallConfiguration(dim3 *__grid_size, dim3 *__block_size,
- size_t *__shared_memory, void *__stream) {
- __omp_kernel_t &__kernel = __current_kernel;
- *__grid_size = __kernel.__grid_size;
- *__block_size = __kernel.__block_size;
- *__shared_memory = __kernel.__shared_memory;
- *((void **)__stream) = __kernel.__stream;
- return 0;
-}
-
-int __tgt_target_kernel(void *Loc, int64_t DeviceId, int32_t NumTeams,
- int32_t ThreadLimit, const void *HostPtr,
- KernelArgsTy *Args);
-
-unsigned llvmLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
- void *args, size_t sharedMem, void *stream) {
- KernelArgsTy Args = {};
- Args.DynCGroupMem = sharedMem;
- Args.UserNumBlocks[0] = gridDim.x;
- Args.UserNumBlocks[1] = gridDim.y;
- Args.UserNumBlocks[2] = gridDim.z;
- Args.UserThreadLimit[0] = blockDim.x;
- Args.UserThreadLimit[1] = blockDim.y;
- Args.UserThreadLimit[2] = blockDim.z;
- Args.ArgPtrs = reinterpret_cast<void **>(args);
- Args.Flags.IsCUDA = true;
- Args.Flags.StrictBlocksAndThreads = true;
- return __tgt_target_kernel(nullptr, 0, gridDim.x, blockDim.x, func, &Args);
-}
-}
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 1831c43cc5f29..6c14572028c7b 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -80,9 +80,6 @@ VERS1.0 {
__tgt_interop_release;
__tgt_target_sync;
__tgt_register_rpc_callback;
- __llvmPushCallConfiguration;
- __llvmPopCallConfiguration;
- llvmLaunchKernel;
local:
*;
};
diff --git a/offload/test/api/kernel_handle.cpp b/offload/test/api/kernel_handle.cpp
new file mode 100644
index 0000000000000..313f1a88aea4d
--- /dev/null
+++ b/offload/test/api/kernel_handle.cpp
@@ -0,0 +1,38 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+#include <omp.h>
+#include <omptarget.h>
+#include <stdio.h>
+
+int main() {
+ int Device = omp_get_num_devices() ? omp_get_default_device()
+ : omp_get_initial_device();
+ if (omp_get_num_devices()) {
+ int X = 0;
+#pragma omp target map(tofrom : X) device(Device)
+ {
+ X = 1;
+ }
+ }
+
+ __tgt_kernel_handle Kernel = reinterpret_cast<void *>(1);
+ // CHECK: null name failed
+ if (__tgt_get_kernel_handle(Device, nullptr, &Kernel) == OFFLOAD_FAIL)
+ printf("null name failed\n");
+
+ // CHECK: null output failed
+ if (__tgt_get_kernel_handle(Device, "missing_kernel", nullptr) ==
+ OFFLOAD_FAIL)
+ printf("null output failed\n");
+
+ // CHECK: missing kernel failed
+ if (__tgt_get_kernel_handle(Device, "missing_kernel", &Kernel) ==
+ OFFLOAD_FAIL &&
+ Kernel == nullptr)
+ printf("missing kernel failed\n");
+
+ __tgt_kernel_properties Properties;
+ // CHECK: null properties failed
+ if (__tgt_get_kernel_properties(nullptr, &Properties) == OFFLOAD_FAIL)
+ printf("null properties failed\n");
+}
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index 5fd7b97dd4216..dc9984ca90be6 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -91,6 +91,11 @@ config.excludes = ['Inputs', 'unit']
# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)
+# language includes
+config.test_language_includes = os.path.join(config.test_source_root, "../languages/include")
+config.test_language_cuda_includes = os.path.join(config.test_language_includes, "cuda")
+config.test_language_hip_includes = os.path.join(config.test_language_includes, "hip")
+
# test_exec_root: The root object directory where output is placed
config.test_exec_root = config.libomptarget_obj_root
@@ -100,6 +105,9 @@ config.test_format = lit.formats.ShTest()
# compiler flags
config.test_flags = " -I " + config.test_source_root + \
" -I " + config.omp_header_directory + \
+ " -I " + config.test_language_includes + \
+ " -I " + config.test_language_cuda_includes + \
+ " -I " + config.test_language_hip_includes + \
" -L " + config.library_dir + \
" -L " + config.llvm_library_intdir + \
" -L " + config.llvm_lib_directory
diff --git a/offload/test/offloading/CUDA/basic_launch.cu b/offload/test/offloading/CUDA/basic_launch.cu
index e017241bb9a74..8be4f336d55d0 100644
--- a/offload/test/offloading/CUDA/basic_launch.cu
+++ b/offload/test/offloading/CUDA/basic_launch.cu
@@ -11,21 +11,17 @@
#include <stdio.h>
-extern "C" {
-void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum);
-void llvm_omp_target_free_shared(void *DevicePtr, int DeviceNum);
-}
-
__global__ void square(int *A) { *A = 42; }
int main(int argc, char **argv) {
- int DevNo = 0;
- int *Ptr = reinterpret_cast<int *>(llvm_omp_target_alloc_shared(4, DevNo));
- *Ptr = 7;
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr:0x.*]], *Ptr: 7
+ int *Ptr;
+ cudaMalloc(&Ptr, 4);
+ printf("Ptr %p\n", Ptr);
+ // CHECK: Ptr [[Ptr:0x.*]]
square<<<1, 1>>>(Ptr);
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr]], *Ptr: 42
- llvm_omp_target_free_shared(Ptr, DevNo);
+ int I = 0;
+ cudaDeviceSynchronize();
+ cudaMemcpy(&I, Ptr, sizeof(int), cudaMemcpyDeviceToHost);
+ printf("I: %i\n", I);
+ // CHECK: I: 42
}
diff --git a/offload/test/offloading/CUDA/basic_launch_blocks_and_threads.cu b/offload/test/offloading/CUDA/basic_launch_blocks_and_threads.cu
index a428e25d82359..a756696e34116 100644
--- a/offload/test/offloading/CUDA/basic_launch_blocks_and_threads.cu
+++ b/offload/test/offloading/CUDA/basic_launch_blocks_and_threads.cu
@@ -11,23 +11,18 @@
#include <stdio.h>
-extern "C" {
-void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum);
-void llvm_omp_target_free_shared(void *DevicePtr, int DeviceNum);
-}
-
__global__ void square(int *A) {
__scoped_atomic_fetch_add(A, 1, __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);
}
int main(int argc, char **argv) {
int DevNo = 0;
- int *Ptr = reinterpret_cast<int *>(llvm_omp_target_alloc_shared(4, DevNo));
- *Ptr = 0;
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr:0x.*]], *Ptr: 0
+ int *Ptr, I;
+ cudaMalloc(&Ptr, 4);
+ printf("Ptr %p\n", Ptr);
+ // CHECK: Ptr [[Ptr:0x.*]]
square<<<7, 6>>>(Ptr);
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr]], *Ptr: 42
- llvm_omp_target_free_shared(Ptr, DevNo);
+ cudaMemcpy(&I, Ptr, sizeof(int), cudaMemcpyDeviceToHost);
+ printf("I: %i\n", I);
+ // CHECK: I: 42
}
diff --git a/offload/test/offloading/CUDA/basic_launch_multi_arg.cu b/offload/test/offloading/CUDA/basic_launch_multi_arg.cu
index db2a1e48371b0..80485b916e344 100644
--- a/offload/test/offloading/CUDA/basic_launch_multi_arg.cu
+++ b/offload/test/offloading/CUDA/basic_launch_multi_arg.cu
@@ -10,11 +10,6 @@
#include <stdio.h>
-extern "C" {
-void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum);
-void llvm_omp_target_free_shared(void *DevicePtr, int DeviceNum);
-}
-
__global__ void square(int *Dst, short Q, int *Src, short P) {
*Dst = (Src[0] + Src[1]) * (Q + P);
Src[0] = Q;
@@ -23,19 +18,19 @@ __global__ void square(int *Dst, short Q, int *Src, short P) {
int main(int argc, char **argv) {
int DevNo = 0;
- int *Ptr = reinterpret_cast<int *>(llvm_omp_target_alloc_shared(4, DevNo));
- int *Src = reinterpret_cast<int *>(llvm_omp_target_alloc_shared(8, DevNo));
- *Ptr = 7;
- Src[0] = -2;
- Src[1] = 8;
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr:0x.*]], *Ptr: 7
- printf("Src: %i : %i\n", Src[0], Src[1]);
- // CHECK: Src: -2 : 8
+ int *Src, *Ptr;
+ cudaMalloc(&Ptr, 4);
+ cudaMalloc(&Src, 8);
+
+ int I = 7;
+ int HostSrc[2] = {-2,8};
+ cudaMemcpy(Ptr, &I, sizeof(int), cudaMemcpyHostToDevice);
+ cudaMemcpy(Src, &HostSrc[0], 2*sizeof(int), cudaMemcpyHostToDevice);
square<<<1, 1>>>(Ptr, 3, Src, 4);
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr]], *Ptr: 42
- printf("Src: %i : %i\n", Src[0], Src[1]);
- // CHECK: Src: 3 : 4
- llvm_omp_target_free_shared(Ptr, DevNo);
+ cudaMemcpy(&I, Ptr, sizeof(int), cudaMemcpyDeviceToHost);
+ cudaMemcpy(&HostSrc[0], Src, 2 * sizeof(int), cudaMemcpyDeviceToHost);
+ printf("I: %i\n", I);
+ // CHECK: I: 42
+ printf("Src: %i, %i\n", HostSrc[0], HostSrc[1]);
+ // CHECK: Src: 3, 4
}
diff --git a/offload/test/offloading/CUDA/launch_tu.cu b/offload/test/offloading/CUDA/launch_tu.cu
index a46472b514a6c..70b9a654b9ae8 100644
--- a/offload/test/offloading/CUDA/launch_tu.cu
+++ b/offload/test/offloading/CUDA/launch_tu.cu
@@ -11,21 +11,17 @@
#include <stdio.h>
-extern "C" {
-void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum);
-void llvm_omp_target_free_shared(void *DevicePtr, int DeviceNum);
-}
-
extern __global__ void square(int *A);
int main(int argc, char **argv) {
int DevNo = 0;
- int *Ptr = reinterpret_cast<int *>(llvm_omp_target_alloc_shared(4, DevNo));
- *Ptr = 7;
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr:0x.*]], *Ptr: 7
+ int *Ptr;
+ cudaMalloc(&Ptr, 4);
+ printf("Ptr %p\n", Ptr);
+ // CHECK: Ptr [[Ptr:0x.*]]
square<<<1, 1>>>(Ptr);
- printf("Ptr %p, *Ptr: %i\n", Ptr, *Ptr);
- // CHECK: Ptr [[Ptr]], *Ptr: 42
- llvm_omp_target_free_shared(Ptr, DevNo);
+ int I;
+ cudaMemcpy(&I, Ptr, sizeof(int), cudaMemcpyDeviceToHost);
+ printf("I: %i\n", I);
+ // CHECK: I: 42
}
diff --git a/offload/test/offloading/CUDA/thread_and_block_id.cu b/offload/test/offloading/CUDA/thread_and_block_id.cu
new file mode 100644
index 0000000000000..7a1b611138f5d
--- /dev/null
+++ b/offload/test/offloading/CUDA/thread_and_block_id.cu
@@ -0,0 +1,41 @@
+// clang-format off
+// RUN: %clang++ %flags -foffload-via-llvm --offload-arch=native %s -o %t
+// RUN: %t | %fcheck-generic
+// RUN: %clang++ %flags -foffload-via-llvm --offload-arch=native %s -o %t -fopenmp
+// RUN: %t | %fcheck-generic
+// clang-format on
+
+// UNSUPPORTED: aarch64-unknown-linux-gnu
+// UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+// UNSUPPORTED: x86_64-unknown-linux-gnu
+// UNSUPPORTED: x86_64-unknown-linux-gnu-LTO
+
+#include <stdio.h>
+#include <stdlib.h>
+
+__global__ void fill(int *A) {
+ int tid = threadIdx.x + blockDim.x * blockIdx.x;
+ A[tid] = 42;
+}
+
+int main(int argc, char **argv) {
+ int NThreads = 128;
+ int NBlocks = 512;
+ int Size = sizeof(int) * NThreads * NBlocks;
+ int *Ptr = (int*)calloc(1, Size);
+ int *DevPtr;
+ cudaMalloc(&DevPtr, Size);
+ cudaMemcpy(DevPtr, Ptr, Size, cudaMemcpyHostToDevice);
+ printf("DevPtr %p\n", DevPtr);
+ // CHECK: DevPtr [[DevPtr:0x.*]]
+ fill<<<NBlocks, NThreads>>>(DevPtr);
+ cudaMemcpy(Ptr, DevPtr, Size, cudaMemcpyDeviceToHost);
+
+ for (int I = 0; I < NBlocks * NThreads; ++I) {
+ if (Ptr[I] == 42)
+ continue;
+ printf("Error at %i: %i vs %i\n", I, Ptr[I], 42);
+ return 1;
+ }
+ return 0;
+}
diff --git a/offload/unittests/OffloadAPI/kernel/olGetKernel.cpp b/offload/unittests/OffloadAPI/kernel/olGetKernel.cpp
new file mode 100644
index 0000000000000..2af2e3e786f26
--- /dev/null
+++ b/offload/unittests/OffloadAPI/kernel/olGetKernel.cpp
@@ -0,0 +1,59 @@
+//===------- Offload API tests - olGetKernel ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+using olGetKernelTest = OffloadProgramTest;
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olGetKernelTest);
+
+using olGetKernelNoProgramTest = OffloadDeviceTest;
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olGetKernelNoProgramTest);
+
+TEST_P(olGetKernelTest, Success) {
+ ol_kernel_handle_t Kernel = nullptr;
+ ASSERT_SUCCESS(olGetKernel(Device, "foo", &Kernel));
+ ASSERT_NE(Kernel, nullptr);
+}
+
+TEST_P(olGetKernelTest, SuccessSameAsSymbol) {
+ ol_kernel_handle_t Kernel = nullptr;
+ ol_symbol_handle_t Symbol = nullptr;
+ ASSERT_SUCCESS(olGetSymbol(Program, "foo", OL_SYMBOL_KIND_KERNEL, &Symbol));
+ ASSERT_SUCCESS(olGetKernel(Device, "foo", &Kernel));
+ ASSERT_EQ(Kernel, Symbol);
+}
+
+TEST_P(olGetKernelTest, InvalidNullDevice) {
+ ol_kernel_handle_t Kernel = nullptr;
+ ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+ olGetKernel(nullptr, "foo", &Kernel));
+}
+
+TEST_P(olGetKernelTest, InvalidNullName) {
+ ol_kernel_handle_t Kernel = nullptr;
+ ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+ olGetKernel(Device, nullptr, &Kernel));
+}
+
+TEST_P(olGetKernelTest, InvalidNullKernelPointer) {
+ ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+ olGetKernel(Device, "foo", nullptr));
+}
+
+TEST_P(olGetKernelTest, InvalidKernelName) {
+ ol_kernel_handle_t Kernel = nullptr;
+ ASSERT_ERROR(OL_ERRC_NOT_FOUND,
+ olGetKernel(Device, "invalid_kernel_name", &Kernel));
+}
+
+TEST_P(olGetKernelNoProgramTest, NoLoadedProgram) {
+ ol_kernel_handle_t Kernel = nullptr;
+ ASSERT_ERROR(OL_ERRC_NOT_FOUND, olGetKernel(Device, "foo", &Kernel));
+}
diff --git a/offload/unittests/OffloadAPI/kernel/olGetKernelProperties.cpp b/offload/unittests/OffloadAPI/kernel/olGetKernelProperties.cpp
new file mode 100644
index 0000000000000..c830a5b94bcec
--- /dev/null
+++ b/offload/unittests/OffloadAPI/kernel/olGetKernelProperties.cpp
@@ -0,0 +1,62 @@
+//===------- Offload API tests - olGetKernelProperties --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+struct olGetKernelPropertiesTest : OffloadProgramTest {
+ void SetUp() override {
+ RETURN_ON_FATAL_FAILURE(OffloadProgramTest::SetUpWith("multiargs"));
+ ASSERT_SUCCESS(olGetKernel(Device, "multiargs", &Kernel));
+ }
+
+ ol_kernel_handle_t Kernel = nullptr;
+};
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olGetKernelPropertiesTest);
+
+using olGetKernelPropertiesGlobalTest = OffloadGlobalTest;
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olGetKernelPropertiesGlobalTest);
+
+TEST_P(olGetKernelPropertiesTest, Success) {
+ ol_kernel_properties_t Properties{};
+ ASSERT_SUCCESS(olGetKernelProperties(Kernel, &Properties));
+
+ ASSERT_EQ(Properties.Version, 1u);
+ ASSERT_TRUE(Properties.ValidFields & OL_KERNEL_PROPERTY_FLAG_NAME);
+ ASSERT_STREQ(Properties.Name, "multiargs");
+ ASSERT_TRUE(Properties.ValidFields & OL_KERNEL_PROPERTY_FLAG_MAX_NUM_THREADS);
+ ASSERT_TRUE(Properties.ValidFields &
+ OL_KERNEL_PROPERTY_FLAG_STATIC_SHARED_MEMORY_SIZE);
+
+ if (Properties.ValidFields & OL_KERNEL_PROPERTY_FLAG_NUM_ARGS)
+ ASSERT_GE(Properties.NumArgs, 3u);
+
+ if (Properties.ValidFields & OL_KERNEL_PROPERTY_FLAG_ARG_SIZES) {
+ ASSERT_NE(Properties.ArgSizes, nullptr);
+ ASSERT_EQ(Properties.NumArgSizes, Properties.NumArgs);
+ for (uint32_t I = 0; I < Properties.NumArgSizes; ++I)
+ ASSERT_GT(Properties.ArgSizes[I], 0u);
+ }
+}
+
+TEST_P(olGetKernelPropertiesTest, InvalidNullKernel) {
+ ol_kernel_properties_t Properties{};
+ ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+ olGetKernelProperties(nullptr, &Properties));
+}
+
+TEST_P(olGetKernelPropertiesTest, InvalidNullProperties) {
+ ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+ olGetKernelProperties(Kernel, nullptr));
+}
+
+TEST_P(olGetKernelPropertiesGlobalTest, InvalidSymbolKind) {
+ ol_kernel_properties_t Properties{};
+ ASSERT_ERROR(OL_ERRC_SYMBOL_KIND, olGetKernelProperties(Global, &Properties));
+}
>From 1d944a68c8d54d543cff0c8e888fe1ed5d68b34d Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 25 Jun 2026 13:36:58 -0700
Subject: [PATCH 02/23] fix CUDA headers being included with -foffload-via-llvm
---
clang/lib/Driver/ToolChains/Clang.cpp | 15 +++++++++++++--
clang/lib/Driver/ToolChains/Cuda.cpp | 4 ++++
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index d1b03d12ae07f..d90108ab0bd2c 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -53,6 +53,7 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include "llvm/TargetParser/ARMTargetParserCommon.h"
#include "llvm/TargetParser/Host.h"
@@ -952,8 +953,13 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
// before we -I or -include anything else, because we must pick up the
// CUDA/HIP/SYCL headers from the particular CUDA/ROCm/SYCL installation,
// rather than from e.g. /usr/local/include.
- if (JA.isOffloading(Action::OFK_Cuda))
+ llvm::errs() << "Is Cuda: " << JA.isOffloading(Action::OFK_Cuda) << "\n";
+ llvm::errs() << "Offloading? " << JA.getOffloadingDeviceKind() << "\n";
+ if (JA.isOffloading(Action::OFK_Cuda)) {
+ llvm::errs() << "going into cuda include args\n";
+ getToolChain().printVerboseInfo(llvm::errs());
getToolChain().AddCudaIncludeArgs(Args, CmdArgs);
+ }
if (JA.isOffloading(Action::OFK_HIP))
getToolChain().AddHIPIncludeArgs(Args, CmdArgs);
if (JA.isOffloading(Action::OFK_SYCL))
@@ -991,7 +997,12 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
} else {
CmdArgs.push_back("__llvm_offload_host.h");
}
- CmdArgs.push_back("-include");
+ SmallString<128> OffloadCudaInclude(D.Dir);
+ llvm::sys::path::append(OffloadCudaInclude, "..", "include", "offload",
+ "cuda");
+ CmdArgs.append({"-internal-isystem", Args.MakeArgString(OffloadCudaInclude),
+ "-include"});
+ // CmdArgs.push_back("-include");
CmdArgs.push_back("cuda_runtime.h");
}
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index e8d9c59f05782..64da0bc3a7b72 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -1050,6 +1050,10 @@ CudaToolChain::GetCXXStdlibType(const ArgList &Args) const {
void CudaToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
ArgStringList &CC1Args) const {
+ if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false))
+ return;
+
HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args);
if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
>From 69d97ac71dd37038d863c6bfc26f518fe3e25a30 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Fri, 26 Jun 2026 10:21:30 -0700
Subject: [PATCH 03/23] fix swapped FreeHost function name
---
offload/languages/include/kernel/DefineLanguageNames.inc | 2 +-
offload/languages/include/kernel/LanguageRuntime.h | 2 +-
offload/languages/include/kernel/UndefineLanguageNames.inc | 2 +-
offload/languages/kernel/src/LanguageRuntime.cpp | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/offload/languages/include/kernel/DefineLanguageNames.inc b/offload/languages/include/kernel/DefineLanguageNames.inc
index 80dc532589ad5..ca9fcdccc5c6e 100644
--- a/offload/languages/include/kernel/DefineLanguageNames.inc
+++ b/offload/languages/include/kernel/DefineLanguageNames.inc
@@ -32,6 +32,6 @@
#define SetDevice COMBINE(LANGUAGE, SetDevice)
#define HostAlloc COMBINE(LANGUAGE, HostAlloc)
#define MallocHost COMBINE(LANGUAGE, MallocHost)
-#define HostFree COMBINE(LANGUAGE, HostFree)
+#define FreeHost COMBINE(LANGUAGE, FreeHost)
#define DriverGetVersion COMBINE(LANGUAGE, DriverGetVersion)
#define GetDeviceProperties COMBINE(LANGUAGE, GetDeviceProperties)
diff --git a/offload/languages/include/kernel/LanguageRuntime.h b/offload/languages/include/kernel/LanguageRuntime.h
index 483eb33acb62f..a213529635ef0 100644
--- a/offload/languages/include/kernel/LanguageRuntime.h
+++ b/offload/languages/include/kernel/LanguageRuntime.h
@@ -81,7 +81,7 @@ Error_t GetDeviceCount(int *Count);
Error_t SetDevice(int DeviceNo);
-Error_t HostFree(void *Ptr);
+Error_t FreeHost(void *Ptr);
Error_t DriverGetVersion(int *Version);
diff --git a/offload/languages/include/kernel/UndefineLanguageNames.inc b/offload/languages/include/kernel/UndefineLanguageNames.inc
index e7cc890eaa2d1..4eaca948b2be4 100644
--- a/offload/languages/include/kernel/UndefineLanguageNames.inc
+++ b/offload/languages/include/kernel/UndefineLanguageNames.inc
@@ -29,6 +29,6 @@
#undef SetDevice
#undef HostAlloc
#undef MallocHost
-#undef HostFree
+#undef FreeHost
#undef DriverGetVersion
#undef GetDeviceProperties
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index 3a08d33e977ec..e32606274a5d4 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -131,7 +131,7 @@ Error_t MallocHost(void **Ptr, size_t Size) {
return HostAlloc(Ptr, Size, /* HostAllocDefault */ 0);
}
-Error_t HostFree(void *Ptr) {
+Error_t FreeHost(void *Ptr) {
ol_result_t Result = olMemFree(Ptr);
return olKConvertResult(Result);
}
>From 8c110f7573ae2f767e4ba0429b0b63794b1b3dce Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Fri, 26 Jun 2026 16:09:11 -0700
Subject: [PATCH 04/23] add cuda_runtime.h headers (unimplemented)
---
offload/languages/include/hip/hip_runtime.h | 8 +++-
.../include/kernel/DefineLanguageNames.inc | 13 ++++++
.../include/kernel/LanguageRuntime.h | 41 +++++++++++++++++++
.../include/kernel/UndefineLanguageNames.inc | 14 +++++++
.../languages/kernel/src/LanguageRuntime.cpp | 25 +++++++++++
5 files changed, 100 insertions(+), 1 deletion(-)
diff --git a/offload/languages/include/hip/hip_runtime.h b/offload/languages/include/hip/hip_runtime.h
index 9762a8a797d34..fd0f31492420a 100644
--- a/offload/languages/include/hip/hip_runtime.h
+++ b/offload/languages/include/hip/hip_runtime.h
@@ -19,7 +19,13 @@
#undef LANGUAGE
-enum hipHostMallocFlag_t { hipHostMallocNonCoherent = 0x80000000 };
+enum hipHostMallocFlag_t : unsigned int {
+ hipHostMallocDefault = hipHostAllocDefault,
+ hipHostMallocPortable = hipHostAllocPortable,
+ hipHostMallocMapped = hipHostAllocMapped,
+ hipHostMallocWriteCombined = hipHostAllocWriteCombined,
+ hipHostMallocNonCoherent = 0x80000000,
+};
hipError_t hipHostMalloc(void **Ptr, size_t Size, unsigned int Flags) {
return hipHostAlloc(Ptr, Size, Flags);
diff --git a/offload/languages/include/kernel/DefineLanguageNames.inc b/offload/languages/include/kernel/DefineLanguageNames.inc
index ca9fcdccc5c6e..fa6727fb06754 100644
--- a/offload/languages/include/kernel/DefineLanguageNames.inc
+++ b/offload/languages/include/kernel/DefineLanguageNames.inc
@@ -31,7 +31,20 @@
#define GetDeviceCount COMBINE(LANGUAGE, GetDeviceCount)
#define SetDevice COMBINE(LANGUAGE, SetDevice)
#define HostAlloc COMBINE(LANGUAGE, HostAlloc)
+#define HostAllocDefault COMBINE(LANGUAGE, HostAllocDefault)
+#define HostAllocPortable COMBINE(LANGUAGE, HostAllocPortable)
+#define HostAllocMapped COMBINE(LANGUAGE, HostAllocMapped)
+#define HostAllocWriteCombined COMBINE(LANGUAGE, HostAllocWriteCombined)
#define MallocHost COMBINE(LANGUAGE, MallocHost)
#define FreeHost COMBINE(LANGUAGE, FreeHost)
#define DriverGetVersion COMBINE(LANGUAGE, DriverGetVersion)
#define GetDeviceProperties COMBINE(LANGUAGE, GetDeviceProperties)
+#define OccupancyMaxPotentialBlockSizeVariableSMem COMBINE(LANGUAGE, OccupancyMaxPotentialBlockSizeVariableSMem)
+#define Stream_t COMBINE(LANGUAGE, Stream_t)
+#define StreamCreate COMBINE(LANGUAGE, StreamCreate)
+#define StreamCreateWithFlags COMBINE(LANGUAGE, StreamCreateWithFlags)
+#define StreamDestroy COMBINE(LANGUAGE, StreamDestroy)
+#define StreamSynchronize COMBINE(LANGUAGE, StreamSynchronize)
+#define StreamCreateWithFlagsFlags COMBINE(LANGUAGE, StreamCreateWithFlagsFlags)
+#define StreamDefault COMBINE(LANGUAGE, StreamDefault)
+#define StreamNonBlocking COMBINE(LANGUAGE, StreamNonBlocking)
diff --git a/offload/languages/include/kernel/LanguageRuntime.h b/offload/languages/include/kernel/LanguageRuntime.h
index a213529635ef0..1bfe3f7e29764 100644
--- a/offload/languages/include/kernel/LanguageRuntime.h
+++ b/offload/languages/include/kernel/LanguageRuntime.h
@@ -7,6 +7,10 @@
*===-----------------------------------------------------------------------===
*/
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
#include <stddef.h>
#include <stdint.h>
@@ -20,6 +24,11 @@ struct DeviceProp_t {
int multiProcessorCount;
int major;
int minor;
+ int ECCEnabled;
+ int pciBusID;
+ int pciDeviceID;
+ int pciDomainID;
+ int memoryBusWidth;
};
enum MemcpyKind {
@@ -30,6 +39,20 @@ enum MemcpyKind {
MemcpyDefault = 4
};
+enum HostAllocFlags : unsigned int {
+ HostAllocDefault = 0x00,
+ HostAllocPortable = 0x01,
+ HostAllocMapped = 0x02,
+ HostAllocWriteCombined = 0x04,
+};
+
+enum StreamCreateWithFlagsFlags : unsigned int {
+ StreamDefault = 0x00,
+ StreamNonBlocking = 0x01,
+};
+
+typedef struct Stream_st *Stream_t;
+
/// Malloc, with type template overlay.
///{
Error_t Malloc(void **Dev_Ptr, size_t Size);
@@ -87,6 +110,24 @@ Error_t DriverGetVersion(int *Version);
Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo);
+template <typename UnaryFunction, class T>
+static inline Error_t OccupancyMaxPotentialBlockSizeVariableSMem(
+ int *minGridSize, int *blockSize, T func,
+ UnaryFunction blockSizeToDynamicSMemSize, int blockSizeLimit = 0) {
+ // TODO: [h15] implement
+ fprintf(stderr,
+ "OccupancyMaxPotentialBlockSizeVariableSMem is not implemented yet");
+ abort();
+}
+
+Error_t StreamCreate(Stream_t *stream);
+
+Error_t StreamCreateWithFlags(Stream_t *stream, unsigned int flags);
+
+Error_t StreamDestroy(Stream_t stream);
+
+Error_t StreamSynchronize(Stream_t stream);
+
///
#if defined(__AMDGPU__) || defined(__NVPTX__)
diff --git a/offload/languages/include/kernel/UndefineLanguageNames.inc b/offload/languages/include/kernel/UndefineLanguageNames.inc
index 4eaca948b2be4..8291ed4c4fb2d 100644
--- a/offload/languages/include/kernel/UndefineLanguageNames.inc
+++ b/offload/languages/include/kernel/UndefineLanguageNames.inc
@@ -28,7 +28,21 @@
#undef GetDeviceCount
#undef SetDevice
#undef HostAlloc
+#undef HostAllocFlags
+#undef HostAllocDefault
+#undef HostAllocPortable
+#undef HostAllocMapped
+#undef HostAllocWriteCombined
#undef MallocHost
#undef FreeHost
#undef DriverGetVersion
#undef GetDeviceProperties
+#undef OccupancyMaxPotentialBlockSizeVariableSMem
+#undef Stream_t
+#undef StreamCreate
+#undef StreamCreateWithFlags
+#undef StreamDestroy
+#undef StreamSynchronize
+#undef StreamCreateWithFlagsFlags
+#undef StreamDefault
+#undef StreamNonBlocking
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index e32606274a5d4..ee48b9f5dd5aa 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -143,6 +143,7 @@ Error_t DriverGetVersion(int *Version) {
}
Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
+ // TODO: [h15] add remaining pci/mem fields
memcpy(&DeviceProp->name[0], "TESTGPU", sizeof("TESTGPU"));
DeviceProp->totalGlobalMem = 1024l * 1024 * 1024 * 40;
DeviceProp->multiProcessorCount = 110;
@@ -150,3 +151,27 @@ Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
DeviceProp->minor = 11;
return Success;
}
+
+Error_t StreamCreate(Stream_t *Stream) {
+ // TODO: [h15] implement
+ fprintf(stderr, LANGUAGE_STR "StreamCreate is not implemented yet");
+ abort();
+}
+
+Error_t StreamCreateWithFlags(Stream_t *Stream, unsigned int Flags) {
+ // TODO: [h15] implement
+ fprintf(stderr, LANGUAGE_STR "StreamCreateWithFlags is not implemented yet");
+ abort();
+}
+
+Error_t StreamDestroy(Stream_t Stream) {
+ // TODO: [h15] implement
+ fprintf(stderr, LANGUAGE_STR "StreamDestroy is not implemented yet");
+ abort();
+}
+
+Error_t StreamSynchronize(Stream_t Stream) {
+ // TODO: [h15] implement
+ fprintf(stderr, LANGUAGE_STR "StreamSynchronize is not implemented yet");
+ abort();
+}
>From f9fae9947b240c9f1c4158e2486b22c2f02f1bc0 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Wed, 1 Jul 2026 11:31:04 -0700
Subject: [PATCH 05/23] fix for host files being given offloading passes
---
clang/lib/Driver/Driver.cpp | 42 +++++++++++----
clang/lib/Driver/ToolChains/Clang.cpp | 26 +++++----
.../llvm_offload_wrappers/__llvm_offload.h | 54 -------------------
.../__llvm_offload_device.h | 54 +++++++++++++++++++
4 files changed, 103 insertions(+), 73 deletions(-)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 7144069ec0207..d2f094f779fda 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -923,8 +923,10 @@ static TripleSet inferOffloadToolchains(Compilation &C,
for (StringRef Arch : A->getValues()) {
if (A->getOption().matches(options::OPT_offload_arch_EQ)) {
if (Arch == "native") {
- for (StringRef Str : getSystemOffloadArchs(C, Kind))
+ for (StringRef Str : getSystemOffloadArchs(C, Kind)) {
+ llvm::errs() << "Inserting into Archs: " << Str.str() << '\n';
Archs.insert(Str.str());
+ }
} else {
Archs.insert(Arch.str());
}
@@ -970,6 +972,7 @@ static TripleSet inferOffloadToolchains(Compilation &C,
llvm::Triple Triple =
OffloadArchToTriple(C.getDefaultToolChain().getTriple(), ID);
+ llvm::errs() << "Detected triple: " << Triple.str() << "\n";
// Make a new argument that dispatches this argument to the appropriate
// toolchain. This is required when we infer it and create potentially
@@ -1014,14 +1017,11 @@ static TripleSet inferOffloadToolchains(Compilation &C,
void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
InputList &Inputs) {
- bool UseLLVMOffload = C.getInputArgs().hasArg(
- options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
- bool IsCuda = !UseLLVMOffload &&
- llvm::any_of(Inputs,
- [](std::pair<types::ID, const llvm::opt::Arg *> &I) {
- return types::isCuda(I.first);
- });
- bool IsHIP = !UseLLVMOffload &&
+ bool IsCuda =
+ llvm::any_of(Inputs, [](std::pair<types::ID, const llvm::opt::Arg *> &I) {
+ return types::isCuda(I.first);
+ });
+ bool IsHIP =
(llvm::any_of(Inputs,
[](std::pair<types::ID, const llvm::opt::Arg *> &I) {
return types::isHIP(I.first);
@@ -1031,13 +1031,22 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
bool IsSYCL = C.getInputArgs().hasFlag(options::OPT_fsycl,
options::OPT_fno_sycl, false);
bool IsOpenMPOffloading =
- UseLLVMOffload ||
(C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
options::OPT_fno_openmp, false) &&
(C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
(C.getInputArgs().hasArg(options::OPT_offload_arch_EQ) &&
!(IsCuda || IsHIP))));
+ bool UseLLVMOffload = C.getInputArgs().hasArg(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
+
+ IsOpenMPOffloading =
+ (IsCuda || IsHIP || IsSYCL || IsOpenMPOffloading) && UseLLVMOffload;
+
+ // We currently don't support any kind of mixed offloading.
+ if (IsOpenMPOffloading)
+ IsCuda = IsHIP = IsSYCL = false;
+
llvm::SmallSet<Action::OffloadKind, 4> Kinds;
const std::pair<bool, Action::OffloadKind> ActiveKinds[] = {
{IsCuda, Action::OFK_Cuda},
@@ -1121,6 +1130,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
auto &TC = getOffloadToolChain(C.getInputArgs(), Kind, Target,
C.getDefaultToolChain().getTriple());
+ llvm::errs() << "Determined TC: " << TC.getArchName() << '\n';
// Emit a warning if the detected CUDA version is too new.
if (Kind == Action::OFK_Cuda) {
@@ -5058,11 +5068,19 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
types::ID InputType = Input.first;
const Arg *InputArg = Input.second;
+ llvm::errs() << "Input Type: " << types::getTypeName(InputType)
+ << ", Input Arg: " << InputArg->getAsString(Args) << '\n';
// Allow the toolchain to be active for unsupported file types if we are "offload-cross-compiling" via llvm-offload.
if (!UseLLVMOffload && ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
(Kind == Action::OFK_HIP && !types::isHIP(InputType))))
continue;
+ // FIX: this effectively disables OpenMP offloading for now
+ if (UseLLVMOffload &&
+ (!types::isCuda(InputType) && !types::isHIP(InputType))) {
+ llvm::errs() << "Not making toolchain\n";
+ continue;
+ }
// Get the product of all bound architectures and toolchains.
SmallVector<std::pair<const ToolChain *, BoundArch>> TCAndArchs;
@@ -6970,6 +6988,9 @@ std::string Driver::GetClPchPath(Compilation &C, StringRef BaseName) const {
const ToolChain &Driver::getOffloadToolChain(
const llvm::opt::ArgList &Args, const Action::OffloadKind Kind,
const llvm::Triple &Target, const llvm::Triple &AuxTarget) const {
+
+ llvm::errs() << "gettting offload TC for Offloading Kind: " << Kind
+ << "and target triple: " << Target.str() << '\n';
std::unique_ptr<ToolChain> &TC =
ToolChains[Target.str() + "/" + AuxTarget.str()];
std::unique_ptr<ToolChain> &HostTC = ToolChains[AuxTarget.str()];
@@ -6994,6 +7015,7 @@ const ToolChain &Driver::getOffloadToolChain(
break;
}
}
+ llvm::errs() << "Determined Toolchain: " << TC->getArchName() << '\n';
if (!TC) {
// Detect the toolchain based off of the target architecture if that failed.
switch (Target.getArch()) {
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index d90108ab0bd2c..e9b0ec2d0499d 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -986,24 +986,32 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
CmdArgs.push_back("__clang_openmp_device_functions.h");
}
- if (Args.hasArg(options::OPT_foffload_via_llvm)) {
+ if (Args.hasArg(options::OPT_foffload_via_llvm) &&
+ (JA.isHostOffloading(C.getActiveOffloadKinds()) ||
+ JA.isDeviceOffloading(Action::OFK_OpenMP))) {
// Add llvm_wrappers/* to our system include path. This lets us wrap
// standard library headers and other headers.
SmallString<128> P(D.ResourceDir);
llvm::sys::path::append(P, "include", "llvm_offload_wrappers");
CmdArgs.append({"-internal-isystem", Args.MakeArgString(P), "-include"});
if (JA.isDeviceOffloading(Action::OFK_OpenMP)) {
+ llvm::errs() << "Including offload_device\n";
CmdArgs.push_back("__llvm_offload_device.h");
} else {
+ llvm::errs() << "Including offload_host\n";
CmdArgs.push_back("__llvm_offload_host.h");
}
- SmallString<128> OffloadCudaInclude(D.Dir);
- llvm::sys::path::append(OffloadCudaInclude, "..", "include", "offload",
- "cuda");
- CmdArgs.append({"-internal-isystem", Args.MakeArgString(OffloadCudaInclude),
- "-include"});
- // CmdArgs.push_back("-include");
- CmdArgs.push_back("cuda_runtime.h");
+ if (llvm::any_of(Inputs, [](const InputInfo &I) {
+ return types::isCuda(I.getType());
+ })) {
+ SmallString<128> OffloadCudaInclude(D.Dir);
+ llvm::sys::path::append(OffloadCudaInclude, "..", "include", "offload",
+ "cuda");
+ CmdArgs.append({"-internal-isystem",
+ Args.MakeArgString(OffloadCudaInclude), "-include"});
+ // CmdArgs.push_back("-include");
+ CmdArgs.push_back("cuda_runtime.h");
+ }
}
// Add -i* options, and automatically translate to
@@ -1185,7 +1193,7 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
Args.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
true) &&
!Args.hasArg(options::OPT_nobuiltininc) &&
- (C.getActiveOffloadKinds() == Action::OFK_OpenMP)) {
+ JA.isDeviceOffloading(Action::OFK_OpenMP)) {
// TODO: CUDA / HIP include their own headers for some common functions
// implemented here. We'll need to clean those up so they do not conflict.
SmallString<128> P(D.ResourceDir);
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
index 7f218e235956c..2898898904e29 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
@@ -9,11 +9,6 @@
#include <stddef.h>
-#pragma push_macro("_OPENMP")
-#define _OPENMP
-#include <gpuintrin.h>
-#pragma pop_macro("_OPENMP")
-
#define __host__ __attribute__((host))
#define __device__ __attribute__((device))
#define __global__ __attribute__((global))
@@ -34,52 +29,3 @@ typedef struct dim3 {
unsigned __llvmPushCallConfiguration(dim3 gridDim, dim3 blockDim,
size_t sharedMem = 0, void *stream = 0);
}
-
-// Make sure nobody can create instances of the coordinate types, take their
-// address, copy, or assign them.
-#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
-#define __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag) \
- __attribute__((device)) __tag() = delete; \
- __attribute__((device)) __tag(const __tag &) = delete; \
- __attribute__((device)) void operator=(const __tag &) const = delete; \
- __attribute__((device)) __tag *operator&() const = delete
-
-#pragma push_macro("__GPU_COORD_BUILTIN")
-#define __GPU_COORD_BUILTIN(__tag, __fx, __fy, __fz) \
- struct __tag { \
- __declspec(property(get = __get_x)) unsigned int x; \
- __declspec(property(get = __get_y)) unsigned int y; \
- __declspec(property(get = __get_z)) unsigned int z; \
- static inline __attribute__((device, always_inline)) unsigned int \
- __get_x() { \
- return __fx; \
- } \
- static inline __attribute__((device, always_inline)) unsigned int \
- __get_y() { \
- return __fy; \
- } \
- static inline __attribute__((device, always_inline)) unsigned int \
- __get_z() { \
- return __fz; \
- } \
- \
- private: \
- __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag); \
- }
-
-__GPU_COORD_BUILTIN(__gpu_builtin_threadIdx_t, __gpu_thread_id_x(),
- __gpu_thread_id_y(), __gpu_thread_id_z());
-__GPU_COORD_BUILTIN(__gpu_builtin_blockIdx_t, __gpu_block_id_x(),
- __gpu_block_id_y(), __gpu_block_id_z());
-__GPU_COORD_BUILTIN(__gpu_builtin_blockDim_t, __gpu_num_threads_x(),
- __gpu_num_threads_y(), __gpu_num_threads_z());
-__GPU_COORD_BUILTIN(__gpu_builtin_gridDim_t, __gpu_num_blocks_x(),
- __gpu_num_blocks_y(), __gpu_num_blocks_z());
-
-#pragma pop_macro("__GPU_COORD_BUILTIN")
-#pragma pop_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
-
-extern const __attribute__((device, weak)) __gpu_builtin_threadIdx_t threadIdx;
-extern const __attribute__((device, weak)) __gpu_builtin_blockIdx_t blockIdx;
-extern const __attribute__((device, weak)) __gpu_builtin_blockDim_t blockDim;
-extern const __attribute__((device, weak)) __gpu_builtin_gridDim_t gridDim;
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h
index 1a813b331515b..5c61289b5d795 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h
@@ -8,3 +8,57 @@
*/
#include "__llvm_offload.h"
+
+#pragma push_macro("_OPENMP")
+#define _OPENMP
+#include <gpuintrin.h>
+#pragma pop_macro("_OPENMP")
+
+// Make sure nobody can create instances of the coordinate types, take their
+// address, copy, or assign them.
+#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+#define __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag) \
+ __attribute__((device)) __tag() = delete; \
+ __attribute__((device)) __tag(const __tag &) = delete; \
+ __attribute__((device)) void operator=(const __tag &) const = delete; \
+ __attribute__((device)) __tag *operator&() const = delete
+
+#pragma push_macro("__GPU_COORD_BUILTIN")
+#define __GPU_COORD_BUILTIN(__tag, __fx, __fy, __fz) \
+ struct __tag { \
+ __declspec(property(get = __get_x)) unsigned int x; \
+ __declspec(property(get = __get_y)) unsigned int y; \
+ __declspec(property(get = __get_z)) unsigned int z; \
+ static inline __attribute__((device, always_inline)) unsigned int \
+ __get_x() { \
+ return __fx; \
+ } \
+ static inline __attribute__((device, always_inline)) unsigned int \
+ __get_y() { \
+ return __fy; \
+ } \
+ static inline __attribute__((device, always_inline)) unsigned int \
+ __get_z() { \
+ return __fz; \
+ } \
+ \
+ private: \
+ __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag); \
+ }
+
+__GPU_COORD_BUILTIN(__gpu_builtin_threadIdx_t, __gpu_thread_id_x(),
+ __gpu_thread_id_y(), __gpu_thread_id_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_blockIdx_t, __gpu_block_id_x(),
+ __gpu_block_id_y(), __gpu_block_id_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_blockDim_t, __gpu_num_threads_x(),
+ __gpu_num_threads_y(), __gpu_num_threads_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_gridDim_t, __gpu_num_blocks_x(),
+ __gpu_num_blocks_y(), __gpu_num_blocks_z());
+
+#pragma pop_macro("__GPU_COORD_BUILTIN")
+#pragma pop_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+
+extern const __attribute__((device, weak)) __gpu_builtin_threadIdx_t threadIdx;
+extern const __attribute__((device, weak)) __gpu_builtin_blockIdx_t blockIdx;
+extern const __attribute__((device, weak)) __gpu_builtin_blockDim_t blockDim;
+extern const __attribute__((device, weak)) __gpu_builtin_gridDim_t gridDim;
>From 7abe5c9e7de21bfe5e8ad3ab6b39c71a3d04b404 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Wed, 1 Jul 2026 17:01:42 -0700
Subject: [PATCH 06/23] fix offload wrappers to only include backend when
actually on device
---
.../llvm_offload_wrappers/__llvm_offload.h | 87 +++++++++++++++++++
.../__llvm_offload_device.h | 53 -----------
2 files changed, 87 insertions(+), 53 deletions(-)
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
index 2898898904e29..256cbc641773b 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
@@ -16,6 +16,24 @@
#define __constant__ __attribute__((constant))
#define __managed__ __attribute__((managed))
+#if defined(__NVPTX__) || defined(__AMDGPU__) || defined(__SPIRV__)
+#define __LLVM_OFFLOAD_HAS_GPU_INTRINSICS 1
+#pragma push_macro("_OPENMP")
+#define _OPENMP
+#include <gpuintrin.h>
+#pragma pop_macro("_OPENMP")
+#else
+#define __LLVM_OFFLOAD_HAS_GPU_INTRINSICS 0
+#endif
+
+#if defined(__CUDA__) || defined(__HIP__) || __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
+#define __LLVM_OFFLOAD_DEVICE_ATTR __attribute__((device))
+#define __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __attribute__((device, weak))
+#else
+#define __LLVM_OFFLOAD_DEVICE_ATTR
+#define __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __attribute__((weak))
+#endif
+
extern "C" {
typedef struct dim3 {
@@ -29,3 +47,72 @@ typedef struct dim3 {
unsigned __llvmPushCallConfiguration(dim3 gridDim, dim3 blockDim,
size_t sharedMem = 0, void *stream = 0);
}
+
+
+// Make sure nobody can create instances of the coordinate types, take their
+// address, copy, or assign them.
+#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+#define __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag) \
+ __LLVM_OFFLOAD_DEVICE_ATTR __tag() = delete; \
+ __LLVM_OFFLOAD_DEVICE_ATTR __tag(const __tag &) = delete; \
+ __LLVM_OFFLOAD_DEVICE_ATTR void operator=(const __tag &) const = delete; \
+ __LLVM_OFFLOAD_DEVICE_ATTR __tag *operator&() const = delete
+
+#pragma push_macro("__GPU_COORD_BUILTIN")
+#if __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
+#pragma push_macro("__GPU_COORD_GETTER")
+#define __GPU_COORD_GETTER(__expr) { return __expr; }
+
+#define __GPU_COORD_BUILTIN(__tag, __fx, __fy, __fz) \
+ struct __tag { \
+ __declspec(property(get = __get_x)) unsigned int x; \
+ __declspec(property(get = __get_y)) unsigned int y; \
+ __declspec(property(get = __get_z)) unsigned int z; \
+ static inline __LLVM_OFFLOAD_DEVICE_ATTR __attribute__((always_inline)) \
+ unsigned int \
+ __get_x() __GPU_COORD_GETTER(__fx) \
+ static inline __LLVM_OFFLOAD_DEVICE_ATTR __attribute__((always_inline)) \
+ unsigned int \
+ __get_y() __GPU_COORD_GETTER(__fy) \
+ static inline __LLVM_OFFLOAD_DEVICE_ATTR __attribute__((always_inline)) \
+ unsigned int \
+ __get_z() __GPU_COORD_GETTER(__fz) \
+ \
+ private: \
+ __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag); \
+ }
+#else
+#define __GPU_COORD_BUILTIN(__tag, __fx, __fy, __fz) \
+ struct __tag { \
+ unsigned int x; \
+ unsigned int y; \
+ unsigned int z; \
+ \
+ private: \
+ __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag); \
+ }
+#endif
+
+__GPU_COORD_BUILTIN(__gpu_builtin_threadIdx_t, __gpu_thread_id_x(),
+ __gpu_thread_id_y(), __gpu_thread_id_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_blockIdx_t, __gpu_block_id_x(),
+ __gpu_block_id_y(), __gpu_block_id_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_blockDim_t, __gpu_num_threads_x(),
+ __gpu_num_threads_y(), __gpu_num_threads_z());
+__GPU_COORD_BUILTIN(__gpu_builtin_gridDim_t, __gpu_num_blocks_x(),
+ __gpu_num_blocks_y(), __gpu_num_blocks_z());
+
+#if __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
+#pragma pop_macro("__GPU_COORD_GETTER")
+#endif
+#pragma pop_macro("__GPU_COORD_BUILTIN")
+#pragma pop_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+#undef __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
+
+extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_threadIdx_t threadIdx;
+extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_blockIdx_t blockIdx;
+extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_blockDim_t blockDim;
+extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_gridDim_t gridDim;
+
+#undef __LLVM_OFFLOAD_DEVICE_ATTR
+#undef __LLVM_OFFLOAD_DEVICE_WEAK_ATTR
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h
index 5c61289b5d795..e588c227a48dd 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload_device.h
@@ -9,56 +9,3 @@
#include "__llvm_offload.h"
-#pragma push_macro("_OPENMP")
-#define _OPENMP
-#include <gpuintrin.h>
-#pragma pop_macro("_OPENMP")
-
-// Make sure nobody can create instances of the coordinate types, take their
-// address, copy, or assign them.
-#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
-#define __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag) \
- __attribute__((device)) __tag() = delete; \
- __attribute__((device)) __tag(const __tag &) = delete; \
- __attribute__((device)) void operator=(const __tag &) const = delete; \
- __attribute__((device)) __tag *operator&() const = delete
-
-#pragma push_macro("__GPU_COORD_BUILTIN")
-#define __GPU_COORD_BUILTIN(__tag, __fx, __fy, __fz) \
- struct __tag { \
- __declspec(property(get = __get_x)) unsigned int x; \
- __declspec(property(get = __get_y)) unsigned int y; \
- __declspec(property(get = __get_z)) unsigned int z; \
- static inline __attribute__((device, always_inline)) unsigned int \
- __get_x() { \
- return __fx; \
- } \
- static inline __attribute__((device, always_inline)) unsigned int \
- __get_y() { \
- return __fy; \
- } \
- static inline __attribute__((device, always_inline)) unsigned int \
- __get_z() { \
- return __fz; \
- } \
- \
- private: \
- __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag); \
- }
-
-__GPU_COORD_BUILTIN(__gpu_builtin_threadIdx_t, __gpu_thread_id_x(),
- __gpu_thread_id_y(), __gpu_thread_id_z());
-__GPU_COORD_BUILTIN(__gpu_builtin_blockIdx_t, __gpu_block_id_x(),
- __gpu_block_id_y(), __gpu_block_id_z());
-__GPU_COORD_BUILTIN(__gpu_builtin_blockDim_t, __gpu_num_threads_x(),
- __gpu_num_threads_y(), __gpu_num_threads_z());
-__GPU_COORD_BUILTIN(__gpu_builtin_gridDim_t, __gpu_num_blocks_x(),
- __gpu_num_blocks_y(), __gpu_num_blocks_z());
-
-#pragma pop_macro("__GPU_COORD_BUILTIN")
-#pragma pop_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
-
-extern const __attribute__((device, weak)) __gpu_builtin_threadIdx_t threadIdx;
-extern const __attribute__((device, weak)) __gpu_builtin_blockIdx_t blockIdx;
-extern const __attribute__((device, weak)) __gpu_builtin_blockDim_t blockDim;
-extern const __attribute__((device, weak)) __gpu_builtin_gridDim_t gridDim;
>From bd17c49242e04b675478b883c1617d4712a59cf0 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 2 Jul 2026 16:16:01 -0700
Subject: [PATCH 07/23] add warpSize attribute
---
.../llvm_offload_wrappers/__llvm_offload.h | 34 ++++++++++++++++++-
.../include/kernel/LanguageRuntime.h | 1 +
.../languages/kernel/src/LanguageRuntime.cpp | 10 ++++--
offload/liboffload/API/Device.td | 1 +
offload/liboffload/src/OffloadImpl.cpp | 3 +-
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 2 +-
offload/plugins-nextgen/cuda/src/rtl.cpp | 2 +-
7 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
index 256cbc641773b..e357b8ecc98bd 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
@@ -48,6 +48,11 @@ unsigned __llvmPushCallConfiguration(dim3 gridDim, dim3 blockDim,
size_t sharedMem = 0, void *stream = 0);
}
+__LLVM_OFFLOAD_DEVICE_ATTR inline void __syncthreads(void) {
+#if __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
+ __gpu_sync_threads();
+#endif
+}
// Make sure nobody can create instances of the coordinate types, take their
// address, copy, or assign them.
@@ -107,12 +112,39 @@ __GPU_COORD_BUILTIN(__gpu_builtin_gridDim_t, __gpu_num_blocks_x(),
#endif
#pragma pop_macro("__GPU_COORD_BUILTIN")
#pragma pop_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
-#undef __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_threadIdx_t threadIdx;
extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_blockIdx_t blockIdx;
extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_blockDim_t blockDim;
extern const __LLVM_OFFLOAD_DEVICE_WEAK_ATTR __gpu_builtin_gridDim_t gridDim;
+// warpSize: reads the actual warp/wavefront size from hardware
+// Uses implicit conversion operator to allow direct use as int.
+#if defined(__NVPTX__) || defined(__AMDGPU__) || defined(__SPIRV__)
+#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+#define __GPU_DISALLOW_BUILTINVAR_ACCESS(__tag) \
+ __LLVM_OFFLOAD_DEVICE_ATTR __tag() = delete; \
+ __LLVM_OFFLOAD_DEVICE_ATTR __tag(const __tag &) = delete; \
+ __LLVM_OFFLOAD_DEVICE_ATTR void operator=(const __tag &) const = delete; \
+ __LLVM_OFFLOAD_DEVICE_ATTR __tag *operator&() const = delete
+
+struct __gpu_builtin_warpSize_t {
+ __LLVM_OFFLOAD_DEVICE_ATTR constexpr operator int() const {
+#if __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
+ return __gpu_num_lanes();
+#endif
+ }
+
+private:
+ __GPU_DISALLOW_BUILTINVAR_ACCESS(__gpu_builtin_warpSize_t);
+};
+#pragma pop_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
+
+// Provide an inline definition instead of just extern declaration
+static const __LLVM_OFFLOAD_DEVICE_ATTR __gpu_builtin_warpSize_t warpSize{};
+#endif
+
+#undef __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
+
#undef __LLVM_OFFLOAD_DEVICE_ATTR
#undef __LLVM_OFFLOAD_DEVICE_WEAK_ATTR
diff --git a/offload/languages/include/kernel/LanguageRuntime.h b/offload/languages/include/kernel/LanguageRuntime.h
index 1bfe3f7e29764..81a68bf299b60 100644
--- a/offload/languages/include/kernel/LanguageRuntime.h
+++ b/offload/languages/include/kernel/LanguageRuntime.h
@@ -21,6 +21,7 @@ enum Error_t : uint32_t {
struct DeviceProp_t {
char name[256];
size_t totalGlobalMem;
+ int warpSize;
int multiProcessorCount;
int major;
int minor;
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index ee48b9f5dd5aa..616f85d03e095 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -144,8 +144,14 @@ Error_t DriverGetVersion(int *Version) {
Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
// TODO: [h15] add remaining pci/mem fields
- memcpy(&DeviceProp->name[0], "TESTGPU", sizeof("TESTGPU"));
- DeviceProp->totalGlobalMem = 1024l * 1024 * 1024 * 40;
+ ol_device_handle_t Device = olKGetDefaultDevice();
+ size_t name_size = 0;
+ olGetDeviceInfoSize(Device, OL_DEVICE_INFO_NAME, &name_size);
+ olGetDeviceInfo(Device, OL_DEVICE_INFO_NAME, name_size, &DeviceProp->name[0]);
+ olGetDeviceInfo(Device, OL_DEVICE_INFO_GLOBAL_MEM_SIZE, sizeof(size_t),
+ &DeviceProp->totalGlobalMem);
+ olGetDeviceInfo(Device, OL_DEVICE_INFO_WARP_SIZE, sizeof(uint32_t),
+ &DeviceProp->warpSize);
DeviceProp->multiProcessorCount = 110;
DeviceProp->major = 47;
DeviceProp->minor = 11;
diff --git a/offload/liboffload/API/Device.td b/offload/liboffload/API/Device.td
index 2f365bc9d6a1b..e25bea7143e08 100644
--- a/offload/liboffload/API/Device.td
+++ b/offload/liboffload/API/Device.td
@@ -44,6 +44,7 @@ def ol_device_info_t : Enum {
TaggedEtor<"MAX_MEM_ALLOC_SIZE", "uint64_t", "The maximum size of memory object allocation in bytes">,
TaggedEtor<"GLOBAL_MEM_SIZE", "uint64_t", "The size of global device memory in bytes">,
TaggedEtor<"WORK_GROUP_LOCAL_MEM_SIZE", "uint64_t", "The maximum size of local shared memory per work group in bytes">,
+ TaggedEtor<"WARP_SIZE", "uint32_t", "Warp size in threads">,
];
list<TaggedEtor> fp_configs = !foreach(type, ["Single", "Double", "Half"], TaggedEtor<type # "_FP_CONFIG", "ol_device_fp_capability_flags_t", type # " precision floating point capability">);
list<TaggedEtor> native_vec_widths = !foreach(type, ["char","short","int","long","float","double","half"], TaggedEtor<"NATIVE_VECTOR_WIDTH_" # type, "uint32_t", "Native vector width for " # type>);
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 34cd0fe170115..caff6630f01c1 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -499,7 +499,8 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
case OL_DEVICE_INFO_SINGLE_FP_CONFIG:
case OL_DEVICE_INFO_DOUBLE_FP_CONFIG:
case OL_DEVICE_INFO_HALF_FP_CONFIG:
- case OL_DEVICE_INFO_MEMORY_CLOCK_RATE: {
+ case OL_DEVICE_INFO_MEMORY_CLOCK_RATE:
+ case OL_DEVICE_INFO_WARP_SIZE: {
// Uint32 values
if (!std::holds_alternative<uint64_t>(Entry->Value))
return makeError(ErrorCode::BACKEND_FAILURE,
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index dda905607c2ed..ee2e9eaca4ab6 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -3313,7 +3313,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
Status = getDeviceAttrRaw(HSA_AGENT_INFO_WAVEFRONT_SIZE, TmpUInt2);
if (Status == HSA_STATUS_SUCCESS)
- Info.add("Wavefront Size", TmpUInt2);
+ Info.add("Wavefront Size", TmpUInt2, "", DeviceInfo::WARP_SIZE);
Status = getDeviceAttrRaw(HSA_AGENT_INFO_WORKGROUP_MAX_SIZE, TmpUInt);
if (Status == HSA_STATUS_SUCCESS)
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index ca234dd97a6ea..421c978e16011 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1139,7 +1139,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_WARP_SIZE, TmpInt);
if (Res == CUDA_SUCCESS)
- Info.add("Warp Size", TmpInt);
+ Info.add("Warp Size", TmpInt, "", DeviceInfo::WARP_SIZE);
Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK, TmpInt);
if (Res == CUDA_SUCCESS)
>From ea8a6cb8a8fd16c6f24aef3939c6fb6baac2115c Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Mon, 6 Jul 2026 10:04:14 -0700
Subject: [PATCH 08/23] fix builtin name conflict
---
.../lib/Headers/llvm_offload_wrappers/__llvm_offload.h | 10 +++++++++-
clang/lib/Headers/nvptxintrin.h | 2 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
index e357b8ecc98bd..f5496d280fb0c 100644
--- a/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
+++ b/clang/lib/Headers/llvm_offload_wrappers/__llvm_offload.h
@@ -48,12 +48,20 @@ unsigned __llvmPushCallConfiguration(dim3 gridDim, dim3 blockDim,
size_t sharedMem = 0, void *stream = 0);
}
-__LLVM_OFFLOAD_DEVICE_ATTR inline void __syncthreads(void) {
+__LLVM_OFFLOAD_DEVICE_ATTR inline void __llvm_offload_syncthreads(void) {
#if __LLVM_OFFLOAD_HAS_GPU_INTRINSICS
__gpu_sync_threads();
#endif
}
+#if defined(__NVPTX__)
+#define __syncthreads() __llvm_offload_syncthreads()
+#else
+__LLVM_OFFLOAD_DEVICE_ATTR inline void __syncthreads(void) {
+ __llvm_offload_syncthreads();
+}
+#endif
+
// Make sure nobody can create instances of the coordinate types, take their
// address, copy, or assign them.
#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
diff --git a/clang/lib/Headers/nvptxintrin.h b/clang/lib/Headers/nvptxintrin.h
index 40704949d8e4b..1f37b53bee057 100644
--- a/clang/lib/Headers/nvptxintrin.h
+++ b/clang/lib/Headers/nvptxintrin.h
@@ -123,7 +123,7 @@ _DEFAULT_FN_ATTRS static __inline__ uint64_t __gpu_ballot(uint64_t __lane_mask,
// Waits for all the threads in the block to converge and issues a fence.
_DEFAULT_FN_ATTRS static __inline__ void __gpu_sync_threads(void) {
- __syncthreads();
+ __nvvm_bar_sync(0);
}
// Waits for all threads in the warp to reconverge for independent scheduling.
>From 1d10856f7d6f1059bb62e63934b4ea3b98c8e2a9 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Tue, 7 Jul 2026 09:04:25 -0700
Subject: [PATCH 09/23] fix rebase issues
---
clang/lib/CodeGen/CGCUDANV.cpp | 64 ++++++++++++++++++----------------
1 file changed, 34 insertions(+), 30 deletions(-)
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 5b46d5c33dfe0..7e2022643ab22 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -189,9 +189,8 @@ class CGNVCUDARuntime : public CGCUDARuntime {
/// Create offloading entries to register globals in RDC mode.
void createOffloadingEntries();
/// For HIP+PGO, emit the per-TU __llvm_profile_sections_<CUID> global.
- /// On the device side, InstrProfiling emits the populated section-bounds
- /// table only when the TU has real profile data. On the host side it is a
- /// placeholder void* shadow stored in
+ /// On the device side it is the populated 7-pointer section-bounds table.
+ /// On the host side it is a placeholder void* shadow stored in
/// OffloadProfShadow, registered later by makeRegisterGlobalsFn (non-RDC)
/// or createOffloadingEntries (RDC) so the runtime can locate the
/// device-side table by name.
@@ -341,26 +340,22 @@ void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
emitDeviceStubBodyLegacy(CGF, Args);
}
-/// Build the input as a sized array of pointers so that it can be launched by
-/// the offloading runtime.
+/// CUDA passes the arguments with a level of indirection. For example, a
+/// (void*, short, void*) is passed as {void **, short *, void **} to the launch
+/// function. For the LLVM/Offload launch we include the number of arguments and their size.
+/// Thus, we pass {{void **, short*, void **}, 3, {sizeof(void*),
+/// sizeof(short), sizeof(void*)}}.
Address CGNVCUDARuntime::prepareKernelArgsLLVMOffload(CodeGenFunction &CGF,
FunctionArgList &Args) {
- SmallVector<llvm::Type *> ArgTypes, KernelLaunchParamsTypes;
- for (auto &Arg : Args)
- ArgTypes.push_back(CGF.ConvertTypeForMem(Arg->getType()));
- llvm::StructType *KernelArgsTy = llvm::StructType::create(ArgTypes);
- llvm::Type *KernelArgsPtrsTy = llvm::ArrayType::get(PtrTy, Args.size());
-
- auto *Int32Ty = CGF.Builder.getInt32Ty();
- KernelLaunchParamsTypes.push_back(Int32Ty);
+ SmallVector<llvm::Type *> KernelLaunchParamsTypes;
+
+ auto *Int64Ty = CGF.Builder.getInt64Ty();
+ KernelLaunchParamsTypes.push_back(PtrTy);
+ KernelLaunchParamsTypes.push_back(Int64Ty);
KernelLaunchParamsTypes.push_back(PtrTy);
llvm::StructType *KernelLaunchParamsTy =
llvm::StructType::create(KernelLaunchParamsTypes);
- Address KernelArgs = CGF.CreateTempAllocaWithoutCast(
- KernelArgsTy, CharUnits::fromQuantity(16), "kernel_args");
- Address KernelArgsPtrs = CGF.CreateTempAllocaWithoutCast(
- KernelArgsPtrsTy, CharUnits::fromQuantity(16), "kernel_args_ptrs");
Address KernelLaunchParams = CGF.CreateTempAllocaWithoutCast(
KernelLaunchParamsTy, CharUnits::fromQuantity(16),
"kernel_launch_params");
@@ -371,17 +366,24 @@ Address CGNVCUDARuntime::prepareKernelArgsLLVMOffload(CodeGenFunction &CGF,
SizeTy, LangAS::Default, CharUnits::fromQuantity(16), "kernel_arg_sizes",
llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size())));
- CGF.Builder.CreateStore(llvm::ConstantInt::get(Int32Ty, Args.size()),
+ CGF.Builder.CreateStore(KernelArgs.emitRawPointer(CGF),
CGF.Builder.CreateStructGEP(KernelLaunchParams, 0));
- CGF.Builder.CreateStore(KernelArgsPtrs.emitRawPointer(CGF),
+ CGF.Builder.CreateStore(llvm::ConstantInt::get(Int64Ty, Args.size()),
CGF.Builder.CreateStructGEP(KernelLaunchParams, 1));
+ CGF.Builder.CreateStore(KernelArgSizes.emitRawPointer(CGF),
+ CGF.Builder.CreateStructGEP(KernelLaunchParams, 2));
for (unsigned i = 0; i < Args.size(); ++i) {
- auto *ArgVal = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[i]));
- Address ArgAddr = CGF.Builder.CreateStructGEP(KernelArgs, i);
- CGF.Builder.CreateStore(ArgVal, ArgAddr);
- CGF.Builder.CreateStore(ArgAddr.emitRawPointer(CGF),
- CGF.Builder.CreateConstArrayGEP(KernelArgsPtrs, i));
+ llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(Args[i]).emitRawPointer(CGF);
+ llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy);
+ CGF.Builder.CreateDefaultAlignedStore(
+ VoidVarPtr, CGF.Builder.CreateConstGEP1_32(
+ PtrTy, KernelArgs.emitRawPointer(CGF), i));
+
+ auto ArgSize = CGM.getDataLayout().getTypeAllocSize( CGM.getTypes().ConvertType(Args[i]->getType()));
+ CGF.Builder.CreateDefaultAlignedStore(
+ llvm::ConstantInt::get(SizeTy, ArgSize), CGF.Builder.CreateConstGEP1_32(
+ PtrTy, KernelArgSizes.emitRawPointer(CGF), i));
}
return KernelLaunchParams;
@@ -1373,9 +1375,13 @@ void CGNVCUDARuntime::createOffloadingEntries() {
}
}
-// For HIP host+device compiles with PGO enabled, emit the host-side shadow for
-// the per-TU __llvm_profile_sections_<CUID> global. Device-side section table
-// emission is owned by InstrProfiling so it can be gated on real profile data.
+// For HIP host+device compiles with PGO enabled, emit the per-TU global
+// __llvm_profile_sections_<CUID>. Device side: a 7-pointer struct holding
+// section start/stop bounds for the names/counters/data sections plus the
+// raw-version variable. Host side: an opaque void* shadow whose only
+// purpose is to give the host-runtime a registered symbol name to look up
+// via hipGetSymbolAddress; the actual device-side data lives in the
+// matching device-side global.
void CGNVCUDARuntime::emitOffloadProfilingSections() {
if (!CGM.getLangOpts().HIP)
return;
@@ -1439,13 +1445,11 @@ void CGNVCUDARuntime::emitOffloadProfilingSections() {
OffloadProfSectionShadows.push_back({Shadow, DeviceName.str()});
};
- // Keep this order in sync with the runtime: data, counters, uniform counters,
- // then names.
+ // Keep this order in sync with the runtime: data, counters, then names.
for (auto &&I : EmittedKernels) {
std::string KernelName = getDeviceSideName(cast<NamedDecl>(I.D));
AddSectionShadow("data", Twine("__profd_") + KernelName);
AddSectionShadow("cnts", Twine("__profc_") + KernelName);
- AddSectionShadow("ucnts", Twine("__llvm_prf_unifcnt_") + KernelName);
AddSectionShadow("names",
Twine(llvm::getInstrProfNamesVarName()) + "_" + CUIDHash);
}
>From 06a188691d269c7824caf998bb6930b2c1d5dcdf Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Wed, 8 Jul 2026 10:25:13 -0700
Subject: [PATCH 10/23] cleanup to now only error at pushcallconf
---
.../clang/Driver/CudaInstallationDetector.h | 3 +
clang/lib/Driver/Driver.cpp | 13 +++-
clang/lib/Driver/ToolChains/Clang.cpp | 53 +++++++-------
clang/lib/Driver/ToolChains/Cuda.cpp | 70 +++++++++++++++----
clang/lib/Driver/ToolChains/HIPAMD.cpp | 3 +-
5 files changed, 94 insertions(+), 48 deletions(-)
diff --git a/clang/include/clang/Driver/CudaInstallationDetector.h b/clang/include/clang/Driver/CudaInstallationDetector.h
index 0fecbfe069ca4..f941c86ac45bb 100644
--- a/clang/include/clang/Driver/CudaInstallationDetector.h
+++ b/clang/include/clang/Driver/CudaInstallationDetector.h
@@ -22,6 +22,7 @@ class CudaInstallationDetector {
const Driver &D;
bool IsValid = false;
CudaVersion Version = CudaVersion::UNKNOWN;
+ llvm::Triple HostTriple;
std::string InstallPath;
std::string BinPath;
std::string LibDevicePath;
@@ -50,6 +51,8 @@ class CudaInstallationDetector {
/// Print information about the detected CUDA installation.
void print(raw_ostream &OS) const;
+ llvm::Triple getHostTriple() const { return HostTriple;}
+
/// Get the detected Cuda install's version.
CudaVersion version() const {
return Version == CudaVersion::NEW ? CudaVersion::PARTIALLY_SUPPORTED
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index d2f094f779fda..30fb162d6604b 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -1040,8 +1040,9 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
bool UseLLVMOffload = C.getInputArgs().hasArg(
options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
- IsOpenMPOffloading =
- (IsCuda || IsHIP || IsSYCL || IsOpenMPOffloading) && UseLLVMOffload;
+ // Switch back to the actual toolchains
+ // IsOpenMPOffloading =
+ // (IsCuda || IsHIP || IsSYCL || IsOpenMPOffloading) && UseLLVMOffload;
// We currently don't support any kind of mixed offloading.
if (IsOpenMPOffloading)
@@ -1133,7 +1134,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
llvm::errs() << "Determined TC: " << TC.getArchName() << '\n';
// Emit a warning if the detected CUDA version is too new.
- if (Kind == Action::OFK_Cuda) {
+ if (Kind == Action::OFK_Cuda && Target.getOS() == llvm::Triple::CUDA) {
auto &CudaInstallation =
static_cast<const toolchains::CudaToolChain &>(TC).CudaInstallation;
if (CudaInstallation.isValid())
@@ -6998,18 +6999,24 @@ const ToolChain &Driver::getOffloadToolChain(
assert(HostTC && "Host toolchain for offloading doesn't exit?");
if (!TC) {
// Detect the toolchain based off of the target operating system.
+ llvm::errs() << Target.getOS() << "\n";
switch (Target.getOS()) {
case llvm::Triple::CUDA:
TC = std::make_unique<toolchains::CudaToolChain>(*this, Target, *HostTC,
Args);
break;
case llvm::Triple::AMDHSA:
+ llvm::errs() << "raa did indeed make it into here\n";
if (Kind == Action::OFK_HIP)
TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
*HostTC, Args);
else if (Kind == Action::OFK_OpenMP)
TC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(*this, Target,
*HostTC, Args);
+ else if (Kind == Action::OFK_Cuda)
+ //TODO: [h15] figure out if this should be a new TC
+ TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
+ *HostTC, Args);
break;
default:
break;
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index e9b0ec2d0499d..ec22a322606da 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -953,10 +953,12 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
// before we -I or -include anything else, because we must pick up the
// CUDA/HIP/SYCL headers from the particular CUDA/ROCm/SYCL installation,
// rather than from e.g. /usr/local/include.
+ bool UsesLLVMOffloading = Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false) && getToolChain().getTriple().getEnvironment() == llvm::Triple::LLVM;
llvm::errs() << "Is Cuda: " << JA.isOffloading(Action::OFK_Cuda) << "\n";
llvm::errs() << "Offloading? " << JA.getOffloadingDeviceKind() << "\n";
- if (JA.isOffloading(Action::OFK_Cuda)) {
- llvm::errs() << "going into cuda include args\n";
+ llvm::errs() << "Triple Env: " << getToolChain().getTriple().getEnvironmentName() << "\n";
+ if (JA.isOffloading(Action::OFK_Cuda) && !UsesLLVMOffloading) {
getToolChain().printVerboseInfo(llvm::errs());
getToolChain().AddCudaIncludeArgs(Args, CmdArgs);
}
@@ -985,33 +987,22 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
CmdArgs.push_back("-include");
CmdArgs.push_back("__clang_openmp_device_functions.h");
}
+ if (UsesLLVMOffloading && JA.isOffloading(Action::OFK_Cuda) &&
+ Args.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
+ true) &&
+ !Args.hasArg(options::OPT_nohipwrapperinc) &&
+ !Args.hasArg(options::OPT_nobuiltininc) &&
+ llvm::any_of(Inputs, [](const InputInfo &I) {
+ return types::isCuda(I.getType());
+ })) {
+ CmdArgs.append({"-include", "__clang_gpu_device_functions.h"});
- if (Args.hasArg(options::OPT_foffload_via_llvm) &&
- (JA.isHostOffloading(C.getActiveOffloadKinds()) ||
- JA.isDeviceOffloading(Action::OFK_OpenMP))) {
- // Add llvm_wrappers/* to our system include path. This lets us wrap
- // standard library headers and other headers.
- SmallString<128> P(D.ResourceDir);
- llvm::sys::path::append(P, "include", "llvm_offload_wrappers");
- CmdArgs.append({"-internal-isystem", Args.MakeArgString(P), "-include"});
- if (JA.isDeviceOffloading(Action::OFK_OpenMP)) {
- llvm::errs() << "Including offload_device\n";
- CmdArgs.push_back("__llvm_offload_device.h");
- } else {
- llvm::errs() << "Including offload_host\n";
- CmdArgs.push_back("__llvm_offload_host.h");
- }
- if (llvm::any_of(Inputs, [](const InputInfo &I) {
- return types::isCuda(I.getType());
- })) {
- SmallString<128> OffloadCudaInclude(D.Dir);
- llvm::sys::path::append(OffloadCudaInclude, "..", "include", "offload",
- "cuda");
- CmdArgs.append({"-internal-isystem",
- Args.MakeArgString(OffloadCudaInclude), "-include"});
- // CmdArgs.push_back("-include");
- CmdArgs.push_back("cuda_runtime.h");
- }
+ SmallString<128> OffloadCudaInclude(D.Dir);
+ llvm::sys::path::append(OffloadCudaInclude, "..", "include", "offload",
+ "cuda");
+ CmdArgs.append({"-internal-isystem", Args.MakeArgString(OffloadCudaInclude),
+ "-include"});
+ CmdArgs.push_back("cuda_runtime.h");
}
// Add -i* options, and automatically translate to
@@ -5183,6 +5174,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
bool IsSYCLDevice = JA.isDeviceOffloading(Action::OFK_SYCL);
bool IsOpenMPDevice = JA.isDeviceOffloading(Action::OFK_OpenMP);
bool IsExtractAPI = isa<ExtractAPIJobAction>(JA);
+ bool UsesLLVMOffloading =
+ Triple.getEnvironment() == llvm::Triple::LLVM ||
+ Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
bool IsDeviceOffloadAction = !(JA.isDeviceOffloading(Action::OFK_None) ||
JA.isDeviceOffloading(Action::OFK_Host));
bool IsHostOffloadingAction =
@@ -5301,7 +5296,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
}
}
- if (IsCuda && !IsCudaDevice) {
+ if (IsCuda && !IsCudaDevice && !UsesLLVMOffloading) {
// We need to figure out which CUDA version we're compiling for, as that
// determines how we load and launch GPU kernels.
auto *CTC = static_cast<const toolchains::CudaToolChain *>(
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 64da0bc3a7b72..849035520abe7 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -146,7 +146,7 @@ void CudaInstallationDetector::WarnIfUnsupportedVersion() const {
CudaInstallationDetector::CudaInstallationDetector(
const Driver &D, const llvm::Triple &HostTriple,
const llvm::opt::ArgList &Args)
- : D(D) {
+ : D(D), HostTriple(HostTriple) {
struct Candidate {
std::string Path;
bool StrictChecking;
@@ -301,9 +301,27 @@ CudaInstallationDetector::CudaInstallationDetector(
void CudaInstallationDetector::AddCudaIncludeArgs(
const ArgList &DriverArgs, ArgStringList &CC1Args) const {
- if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false))
+ if (HostTriple.getEnvironment() == llvm::Triple::LLVM ||
+ DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false)) {
+ if (DriverArgs.hasFlag(options::OPT_offload_inc,
+ options::OPT_no_offload_inc, true) &&
+ !DriverArgs.hasArg(options::OPT_nohipwrapperinc) &&
+ !DriverArgs.hasArg(options::OPT_nobuiltininc)){
+ CC1Args.append({"-include", "__clang_gpu_device_functions.h"});
+
+ SmallString<128> CudaIncludePath(D.ResourceDir);
+ llvm::sys::path::append(CudaIncludePath, "..", "..", "..");
+ llvm::sys::path::append(CudaIncludePath, "include", "offload", "cuda");
+ CC1Args.push_back("-internal-isystem");
+ CC1Args.push_back(DriverArgs.MakeArgString(CudaIncludePath));
+ CC1Args.append({"-include", "cuda_runtime.h"});
+ }
return;
+ }
+ // if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ // options::OPT_fno_offload_via_llvm, false))
+ // return;
if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
// Add cuda_wrappers/* to our system include path. This lets us wrap
@@ -401,7 +419,7 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
static_cast<const toolchains::NVPTXToolChain &>(getToolChain());
bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
+ options::OPT_fno_offload_via_llvm, false);
assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
BoundArch GPUArch;
@@ -548,7 +566,7 @@ void NVPTX::FatBinary::ConstructJob(Compilation &C, const JobAction &JA,
const auto &TC =
static_cast<const toolchains::CudaToolChain &>(getToolChain());
bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
+ options::OPT_fno_offload_via_llvm, false);
assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
ArgStringList CmdArgs;
@@ -598,7 +616,7 @@ void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
ArgStringList CmdArgs;
bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
+ options::OPT_fno_offload_via_llvm, false);
assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
assert((Output.isFilename() || Output.isNothing()) && "Invalid output.");
@@ -918,6 +936,17 @@ void CudaToolChain::addClangTargetOptions(
DriverArgs.hasArg(options::OPT_S))
return;
+ // For now, we don't use any Offload/OpenMP device runtime when we offload
+ // CUDA via LLVM/Offload. We should split the Offload/OpenMP device runtime
+ // and include the "generic" (or CUDA-specific) parts.
+ // When using LLVM offload path, we use __clang_gpu_device_functions.h
+ // instead of libdevice.
+ bool UseLLVMOffload = getTriple().getEnvironment() == llvm::Triple::LLVM ||
+ DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+ if (UseLLVMOffload)
+ return;
+
std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch);
if (LibDeviceFile.empty()) {
getDriver().Diag(diag::err_drv_no_cuda_libdevice) << GpuArch;
@@ -927,13 +956,6 @@ void CudaToolChain::addClangTargetOptions(
CC1Args.push_back("-mlink-builtin-bitcode");
CC1Args.push_back(DriverArgs.MakeArgString(LibDeviceFile));
- // For now, we don't use any Offload/OpenMP device runtime when we offload
- // CUDA via LLVM/Offload. We should split the Offload/OpenMP device runtime
- // and include the "generic" (or CUDA-specific) parts.
- if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false))
- return;
-
clang::CudaVersion CudaInstallationVersion = CudaInstallation.version();
if (CudaInstallationVersion >= CudaVersion::UNKNOWN)
@@ -974,9 +996,27 @@ llvm::DenormalMode CudaToolChain::getDefaultDenormalModeForType(
void CudaToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,
ArgStringList &CC1Args) const {
- if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false))
+ if (getTriple().getEnvironment() == llvm::Triple::LLVM ||
+ DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false)) {
+ if (DriverArgs.hasFlag(options::OPT_offload_inc,
+ options::OPT_no_offload_inc, true) &&
+ !DriverArgs.hasArg(options::OPT_nohipwrapperinc) &&
+ !DriverArgs.hasArg(options::OPT_nobuiltininc)){
+ CC1Args.append({"-include", "__clang_gpu_device_functions.h"});
+
+ SmallString<128> CudaIncludePath(getDriver().ResourceDir);
+ llvm::sys::path::append(CudaIncludePath, "..", "..", "..");
+ llvm::sys::path::append(CudaIncludePath, "include", "offload", "cuda");
+ CC1Args.push_back("-internal-isystem");
+ CC1Args.push_back(DriverArgs.MakeArgString(CudaIncludePath));
+ CC1Args.append({"-include", "cuda_runtime.h"});
+ }
return;
+ }
+ // if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ // options::OPT_fno_offload_via_llvm, false))
+ // return;
// Check our CUDA version if we're going to include the CUDA headers.
if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index 78a970c91167b..abcba23a370fe 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -235,7 +235,8 @@ HIPAMDToolChain::HIPAMDToolChain(const Driver &D, const llvm::Triple &Triple,
void HIPAMDToolChain::addClangTargetOptions(
const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
BoundArch BA, Action::OffloadKind DeviceOffloadingKind) const {
- assert(DeviceOffloadingKind == Action::OFK_HIP &&
+ bool UsesLLVMOffloading = DriverArgs.hasFlag(options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false) || getTriple().getEnvironment() == llvm::Triple::LLVM;
+ assert((DeviceOffloadingKind == Action::OFK_HIP || UsesLLVMOffloading) &&
"Only HIP offloading kinds are supported for GPUs.");
CC1Args.append({"-fcuda-is-device", "-fno-threadsafe-statics"});
>From 614eefcd14c35ae68dcac44c3e3e6de2ecfa9c61 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 9 Jul 2026 08:13:07 -0700
Subject: [PATCH 11/23] runs Cuda on AMD hw
---
clang/include/clang/Driver/CommonArgs.h | 4 ++++
clang/lib/Driver/Driver.cpp | 18 +++++++-----------
clang/lib/Driver/ToolChains/Clang.cpp | 4 ++--
clang/lib/Driver/ToolChains/CommonArgs.cpp | 12 ++++++++++++
clang/lib/Driver/ToolChains/Gnu.cpp | 1 +
clang/lib/Headers/__clang_gpu_builtin_vars.h | 18 ++++++++++++++++++
.../ClangLinkerWrapper.cpp | 17 ++++++++++++++++-
llvm/lib/Object/OffloadBinary.cpp | 2 ++
8 files changed, 62 insertions(+), 14 deletions(-)
diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index 14a83ff7beae4..21aef962c2828 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -131,6 +131,10 @@ void addArchSpecificRPath(const ToolChain &TC, const llvm::opt::ArgList &Args,
void addOpenMPRuntimeLibraryPath(const ToolChain &TC,
const llvm::opt::ArgList &Args,
llvm::opt::ArgStringList &CmdArgs);
+
+bool addLLVMOffloadingRuntime(const Compilation &C, llvm::opt::ArgStringList &CmdArgs,
+ const ToolChain &TC, const llvm::opt::ArgList &Args);
+
/// Returns true, if an OpenMP runtime has been added.
bool addOpenMPRuntime(const Compilation &C, llvm::opt::ArgStringList &CmdArgs,
const ToolChain &TC, const llvm::opt::ArgList &Args,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 30fb162d6604b..e2358eefca272 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -5047,7 +5047,7 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
getFinalPhase(Args) == phases::Preprocess))
return HostAction;
- bool UseLLVMOffload = Args.hasArg(
+ bool UsesLLVMOffloading = Args.hasArg(
options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
ActionList OffloadActions;
@@ -5057,6 +5057,8 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
Action::OFK_OpenMP, Action::OFK_Cuda, Action::OFK_HIP, Action::OFK_SYCL};
for (Action::OffloadKind Kind : OffloadKinds) {
+ llvm::errs() << "Offload Kind: " << Action::GetOffloadKindName(Kind) \
+ << '\n';
SmallVector<const ToolChain *, 2> ToolChains;
ActionList DeviceActions;
@@ -5073,15 +5075,9 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
<< ", Input Arg: " << InputArg->getAsString(Args) << '\n';
// Allow the toolchain to be active for unsupported file types if we are "offload-cross-compiling" via llvm-offload.
- if (!UseLLVMOffload && ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
+ if (!UsesLLVMOffloading && ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
(Kind == Action::OFK_HIP && !types::isHIP(InputType))))
continue;
- // FIX: this effectively disables OpenMP offloading for now
- if (UseLLVMOffload &&
- (!types::isCuda(InputType) && !types::isHIP(InputType))) {
- llvm::errs() << "Not making toolchain\n";
- continue;
- }
// Get the product of all bound architectures and toolchains.
SmallVector<std::pair<const ToolChain *, BoundArch>> TCAndArchs;
@@ -5203,7 +5199,7 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
return HostAction;
OffloadAction::DeviceDependences DDep;
- if (C.isOffloadingHostKind(Action::OFK_Cuda) &&
+ if (!UsesLLVMOffloading && C.isOffloadingHostKind(Action::OFK_Cuda) &&
!Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, false)) {
// If we are not in RDC-mode we just emit the final CUDA fatbinary for
// each translation unit without requiring any linking.
@@ -5211,7 +5207,7 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
C.MakeAction<LinkJobAction>(OffloadActions, types::TY_CUDA_FATBIN);
DDep.add(*FatbinAction, *C.getSingleOffloadToolChain<Action::OFK_Cuda>(),
/*BA=*/{}, Action::OFK_Cuda);
- } else if (HIPNoRDC && offloadDeviceOnly()) {
+ } else if (!UsesLLVMOffloading && HIPNoRDC && offloadDeviceOnly()) {
// If we are in device-only non-RDC-mode we just emit the final HIP
// fatbinary for each translation unit, linking each input individually.
Action *FatbinAction =
@@ -5219,7 +5215,7 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
DDep.add(*FatbinAction,
*C.getOffloadToolChains<Action::OFK_HIP>().first->second,
/*BA=*/{}, Action::OFK_HIP);
- } else if (HIPNoRDC) {
+ } else if (!UsesLLVMOffloading && HIPNoRDC) {
// Host + device assembly: defer to clang-offload-bundler (see
// BuildActions).
if (HIPAsmBundleDeviceOut &&
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index ec22a322606da..37f435e635e91 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -8290,11 +8290,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
// Host-side offloading compilation receives all device-side outputs. Include
// them in the host compilation depending on the target. If the host inputs
// are not empty we use the new-driver scheme, otherwise use the old scheme.
- if ((IsCuda || IsHIP) && CudaDeviceInput) {
+ if ((IsCuda || IsHIP) && !UsesLLVMOffloading && CudaDeviceInput) {
CmdArgs.push_back("-fcuda-include-gpubinary");
CmdArgs.push_back(CudaDeviceInput->getFilename());
} else if (!HostOffloadingInputs.empty()) {
- if ((IsCuda || IsHIP) && !IsRDCMode) {
+ if ((IsCuda || IsHIP) && !UsesLLVMOffloading && !IsRDCMode) {
assert(HostOffloadingInputs.size() == 1 && "Only one input expected");
CmdArgs.push_back("-fcuda-include-gpubinary");
CmdArgs.push_back(HostOffloadingInputs.front().getFilename());
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index e89dd00cebd04..a747071f8e6c1 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1454,6 +1454,18 @@ void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args,
}
}
+bool tools::addLLVMOffloadingRuntime(const Compilation &C, ArgStringList &CmdArgs,
+ const ToolChain &TC, const ArgList &Args) {
+
+ if (!Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false))
+ return false;
+
+ CmdArgs.push_back("-lLLVMhip64");
+ CmdArgs.push_back("-lLLVMcudart");
+ return true;
+}
+
bool tools::addOpenMPRuntime(const Compilation &C, ArgStringList &CmdArgs,
const ToolChain &TC, const ArgList &Args,
bool ForceStaticHostRuntime, bool IsOffloadingHost,
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index b58c10145e7f3..5e43699f43f4c 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -510,6 +510,7 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
// FIXME: Does this really make sense for all GNU toolchains?
WantPthread = true;
+ addLLVMOffloadingRuntime(C, CmdArgs, ToolChain, Args);
AddRunTimeLibs(ToolChain, D, CmdArgs, Args);
// LLVM support for atomics on 32-bit SPARC V8+ is incomplete, so
diff --git a/clang/lib/Headers/__clang_gpu_builtin_vars.h b/clang/lib/Headers/__clang_gpu_builtin_vars.h
index b80248dcd2be3..80412ea36c261 100644
--- a/clang/lib/Headers/__clang_gpu_builtin_vars.h
+++ b/clang/lib/Headers/__clang_gpu_builtin_vars.h
@@ -6,6 +6,8 @@
//
//===-----------------------------------------------------------------------===
+#include <stddef.h>
+
#ifndef __CLANG_GPU_BUILTIN_VARS_H__
#define __CLANG_GPU_BUILTIN_VARS_H__
@@ -20,6 +22,22 @@ static inline __attribute__((device)) const struct {
}
} warpSize{};
+extern "C" {
+
+typedef struct dim3 {
+ dim3() {}
+ dim3(unsigned x) : x(x) {}
+ unsigned x = 0, y = 0, z = 0;
+} dim3;
+
+// TODO: For some reason the CUDA device compilation requires this declaration
+// to be present on the device while it is only used on the host.
+unsigned __llvmPushCallConfiguration(dim3 gridDim, dim3 blockDim,
+ size_t sharedMem = 0, void *stream = 0);
+unsigned __llvmLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
+ void **args, size_t sharedMem = 0, void *stream = 0);
+}
+
// Make sure nobody can create instances of the coordinate types, take their
// address, copy, or assign them.
#pragma push_macro("__GPU_DISALLOW_BUILTINVAR_ACCESS")
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 5abda37ada7fb..3b14168b7b16d 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -139,6 +139,13 @@ static bool CanonicalPrefixes = true;
using OffloadingImage = OffloadBinary::OffloadingImage;
+static bool usesLLVMOffloadWrapper(ArrayRef<OffloadingImage> Images) {
+ return llvm::any_of(Images, [](const OffloadingImage &Image) {
+ return Triple(Image.StringData.lookup("triple")).getEnvironment() ==
+ Triple::LLVM;
+ });
+}
+
namespace llvm {
// Provide DenseMapInfo so that OffloadKind can be used in a DenseMap.
template <> struct DenseMapInfo<OffloadKind> {
@@ -792,6 +799,8 @@ wrapDeviceImages(ArrayRef<std::unique_ptr<MemoryBuffer>> Buffers,
M.setTargetTriple(Triple(
Args.getLastArgValue(OPT_host_triple_EQ, sys::getDefaultTargetTriple())));
+ llvm::errs() << "Switching on offload kind: " << getOffloadKindName(Kind)
+ << "\n";
switch (Kind) {
case OFK_OpenMP:
if (Error Err = offloading::wrapOpenMPBinaries(
@@ -971,6 +980,11 @@ Expected<SmallVector<std::unique_ptr<MemoryBuffer>>>
bundleLinkedOutput(ArrayRef<OffloadingImage> Images, const ArgList &Args,
OffloadKind Kind) {
llvm::TimeTraceScope TimeScope("Bundle linked output");
+ llvm::errs() << "Bundling on offload kind: " << getOffloadKindName(Kind)
+ << "\n";
+ if (usesLLVMOffloadWrapper(Images))
+ return bundleOpenMP(Images);
+
switch (Kind) {
case OFK_OpenMP:
return (Verbose && SaveTemps) ? bundleOpenMPVerbose(Images)
@@ -1214,7 +1228,8 @@ linkAndWrapDeviceFiles(ArrayRef<SmallVector<OffloadFile>> LinkerInputFiles,
continue;
}
- auto OutputOrErr = wrapDeviceImages(*BundledImagesOrErr, Args, Kind);
+ OffloadKind WrapperKind = usesLLVMOffloadWrapper(Input) ? OFK_OpenMP : Kind;
+ auto OutputOrErr = wrapDeviceImages(*BundledImagesOrErr, Args, WrapperKind);
if (!OutputOrErr)
return OutputOrErr.takeError();
WrappedOutput.push_back(*OutputOrErr);
diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp
index e5c9f4dae131d..43317a2b5552d 100644
--- a/llvm/lib/Object/OffloadBinary.cpp
+++ b/llvm/lib/Object/OffloadBinary.cpp
@@ -403,6 +403,8 @@ OffloadKind object::getOffloadKind(StringRef Name) {
StringRef object::getOffloadKindName(OffloadKind Kind) {
switch (Kind) {
+ case OFK_LLVM:
+ return "llvm";
case OFK_OpenMP:
return "openmp";
case OFK_Cuda:
>From 7c43c41310ad5f1d93c10fb0f0e0b00b79628379 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 9 Jul 2026 15:41:29 -0700
Subject: [PATCH 12/23] runs CUDA on NVIDIA hw
---
clang/lib/Driver/Driver.cpp | 11 +++++++----
clang/lib/Driver/ToolChains/Clang.cpp | 7 +++----
clang/lib/Driver/ToolChains/Cuda.cpp | 23 ++++++++++++-----------
3 files changed, 22 insertions(+), 19 deletions(-)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index e2358eefca272..b24e21c524c04 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -5075,8 +5075,8 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
<< ", Input Arg: " << InputArg->getAsString(Args) << '\n';
// Allow the toolchain to be active for unsupported file types if we are "offload-cross-compiling" via llvm-offload.
- if (!UsesLLVMOffloading && ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
- (Kind == Action::OFK_HIP && !types::isHIP(InputType))))
+ if ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
+ (Kind == Action::OFK_HIP && !types::isHIP(InputType)))
continue;
// Get the product of all bound architectures and toolchains.
@@ -5169,9 +5169,12 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
OffloadAction::DeviceDependences DDep;
DDep.add(*A, *TCAndArch->first, TCAndArch->second, Kind);
- // Compiling CUDA in non-RDC mode uses the PTX output if available.
+ // The legacy CUDA fatbinary path can include PTX alongside the cubin.
+ // The LLVM offload wrapper path feeds these images through a device
+ // linker first, and clang-nvlink-wrapper does not accept PTX as input.
for (Action *Input : A->getInputs())
- if (Kind == Action::OFK_Cuda && A->getType() == types::TY_Object &&
+ if (!UsesLLVMOffloading && Kind == Action::OFK_Cuda &&
+ A->getType() == types::TY_Object &&
!Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc,
false))
DDep.add(*Input, *TCAndArch->first, TCAndArch->second, Kind);
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 37f435e635e91..32786dbd60fe2 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -987,14 +987,13 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
CmdArgs.push_back("-include");
CmdArgs.push_back("__clang_openmp_device_functions.h");
}
+ bool isCudaInput = llvm::any_of(
+ Inputs, [](const InputInfo &I) { return types::isCuda(I.getType()); });
if (UsesLLVMOffloading && JA.isOffloading(Action::OFK_Cuda) &&
Args.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
true) &&
!Args.hasArg(options::OPT_nohipwrapperinc) &&
- !Args.hasArg(options::OPT_nobuiltininc) &&
- llvm::any_of(Inputs, [](const InputInfo &I) {
- return types::isCuda(I.getType());
- })) {
+ !Args.hasArg(options::OPT_nobuiltininc) && isCudaInput) {
CmdArgs.append({"-include", "__clang_gpu_device_functions.h"});
SmallString<128> OffloadCudaInclude(D.Dir);
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 849035520abe7..0a91c411d528c 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -418,9 +418,9 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
const auto &TC =
static_cast<const toolchains::NVPTXToolChain &>(getToolChain());
- bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
- assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
+ bool UsesLLVMOffloading = Args.hasFlag(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
+ assert((UsesLLVMOffloading || TC.getTriple().isNVPTX()) && "Wrong platform");
BoundArch GPUArch;
// If this is a CUDA action we need to extract the device architecture
@@ -443,7 +443,7 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
"Device action expected to have an architecture.");
// Check that our installation's ptxas supports gpu_arch.
- if (!UseLLVMOffload && !Args.hasArg(options::OPT_no_cuda_version_check)) {
+ if (!UsesLLVMOffloading && !Args.hasArg(options::OPT_no_cuda_version_check)) {
TC.CudaInstallation.CheckCudaVersionSupportsArch(GPUArch.Arch);
}
@@ -516,7 +516,8 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
/*Default=*/true);
else if (JA.isOffloading(Action::OFK_Cuda))
// In CUDA we generate relocatable code by default.
- Relocatable = Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc,
+ Relocatable = UsesLLVMOffloading ||
+ Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc,
/*Default=*/false);
else
// Otherwise, we are compiling directly and should create linkable output.
@@ -565,9 +566,9 @@ void NVPTX::FatBinary::ConstructJob(Compilation &C, const JobAction &JA,
const char *LinkingOutput) const {
const auto &TC =
static_cast<const toolchains::CudaToolChain &>(getToolChain());
- bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
- assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
+ bool UsesLLVMOffloading = Args.hasFlag(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
+ assert((UsesLLVMOffloading || TC.getTriple().isNVPTX()) && "Wrong platform");
ArgStringList CmdArgs;
if (TC.CudaInstallation.version() <= CudaVersion::CUDA_100)
@@ -615,9 +616,9 @@ void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
static_cast<const toolchains::NVPTXToolChain &>(getToolChain());
ArgStringList CmdArgs;
- bool UseLLVMOffload = Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
- assert((UseLLVMOffload || TC.getTriple().isNVPTX()) && "Wrong platform");
+ bool UsesLLVMOffloading = Args.hasFlag(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
+ assert((UsesLLVMOffloading || TC.getTriple().isNVPTX()) && "Wrong platform");
assert((Output.isFilename() || Output.isNothing()) && "Invalid output.");
if (Output.isFilename()) {
>From 3c08c622073f84009592e19317770974f4646c29 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 9 Jul 2026 16:34:32 -0700
Subject: [PATCH 13/23] fix __syncthreads error on AMD hw
---
clang/lib/Headers/__clang_gpu_device_functions.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/Headers/__clang_gpu_device_functions.h b/clang/lib/Headers/__clang_gpu_device_functions.h
index 82d02839679bf..a5fb1253926f3 100644
--- a/clang/lib/Headers/__clang_gpu_device_functions.h
+++ b/clang/lib/Headers/__clang_gpu_device_functions.h
@@ -329,7 +329,7 @@ __GPU_DEVICE__ int __fns(unsigned int __mask, unsigned int __base,
//===----------------------------------------------------------------------===//
// CUDA provides __syncthreads as an NVPTX compiler builtin directly.
-#if !defined(__CUDA__)
+#if !defined(__NVPTX__)
__GPU_DEVICE__ void __syncthreads(void) { __gpu_sync_threads(); }
#endif
>From cf2610787e3dbc6f66ee83a4fab556811044169c Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Fri, 10 Jul 2026 14:14:02 -0700
Subject: [PATCH 14/23] add stream with flags stub
---
.../include/kernel/DefineLanguageNames.inc | 1 +
.../include/kernel/LanguageRuntime.h | 1 +
.../include/kernel/UndefineLanguageNames.inc | 1 +
.../kernel/src/LanguageRegistration.cpp | 7 ++-
.../languages/kernel/src/LanguageRuntime.cpp | 59 ++++++++++++++-----
5 files changed, 50 insertions(+), 19 deletions(-)
diff --git a/offload/languages/include/kernel/DefineLanguageNames.inc b/offload/languages/include/kernel/DefineLanguageNames.inc
index fa6727fb06754..d74703af0f787 100644
--- a/offload/languages/include/kernel/DefineLanguageNames.inc
+++ b/offload/languages/include/kernel/DefineLanguageNames.inc
@@ -18,6 +18,7 @@
#define Memcpy COMBINE(LANGUAGE, Memcpy)
#define DeviceSynchronize COMBINE(LANGUAGE, DeviceSynchronize)
#define Success COMBINE(LANGUAGE, Success)
+#define ErrorInvalidValue COMBINE(LANGUAGE, ErrorInvalidValue)
#define MemcpyKind COMBINE(LANGUAGE, MemcpyKind)
#define MemcpyHostToHost COMBINE(LANGUAGE, MemcpyHostToHost)
#define MemcpyHostToDevice COMBINE(LANGUAGE, MemcpyHostToDevice)
diff --git a/offload/languages/include/kernel/LanguageRuntime.h b/offload/languages/include/kernel/LanguageRuntime.h
index 81a68bf299b60..9469105142ffb 100644
--- a/offload/languages/include/kernel/LanguageRuntime.h
+++ b/offload/languages/include/kernel/LanguageRuntime.h
@@ -16,6 +16,7 @@
enum Error_t : uint32_t {
Success = 0,
+ ErrorInvalidValue = 1,
};
struct DeviceProp_t {
diff --git a/offload/languages/include/kernel/UndefineLanguageNames.inc b/offload/languages/include/kernel/UndefineLanguageNames.inc
index 8291ed4c4fb2d..19b8eebe3994d 100644
--- a/offload/languages/include/kernel/UndefineLanguageNames.inc
+++ b/offload/languages/include/kernel/UndefineLanguageNames.inc
@@ -15,6 +15,7 @@
#undef Memcpy
#undef DeviceSynchronize
#undef Success
+#undef ErrorInvalidValue
#undef MemcpyKind
#undef MemcpyHostToHost
#undef MemcpyHostToDevice
diff --git a/offload/languages/kernel/src/LanguageRegistration.cpp b/offload/languages/kernel/src/LanguageRegistration.cpp
index d0c7cae5d007a..6171e6a92afd6 100644
--- a/offload/languages/kernel/src/LanguageRegistration.cpp
+++ b/offload/languages/kernel/src/LanguageRegistration.cpp
@@ -194,8 +194,9 @@ extern "C" {
void __llvmRegisterFunction(const char *Binary, const char *KernelID,
char *KernelName, const char *KernelName1, int,
uint3 *, uint3 *, dim3 *, dim3 *, int *) {
- printf("%s :: %p :: %p : %s : %s \n", __PRETTY_FUNCTION__, Binary, KernelID,
- KernelName, KernelName1);
+ // printf("%s :: %p :: %p : %s : %s \n", __PRETTY_FUNCTION__, Binary,
+ // KernelID,
+ // KernelName, KernelName1);
ol_symbol_handle_t Kernel;
ol_program_handle_t Program = olKGetProgram(Binary);
ol_result_t Result = olGetSymbol(
@@ -206,7 +207,7 @@ void __llvmRegisterFunction(const char *Binary, const char *KernelID,
abort();
}
- printf("K %p : %p\n", KernelID, Kernel);
+ // printf("K %p : %p\n", KernelID, Kernel);
olKRegisterKernel(KernelID, Kernel);
}
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index 616f85d03e095..1ae4201df2b62 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -28,7 +28,16 @@
#define STR(X) #X
#define LANGUAGE_STR STR(LANGUAGE)
-Error_t olKConvertResult(ol_result_t Result) { return Success; }
+static Error_t olKConvertResult(ol_result_t Result) {
+ if (Result == OL_SUCCESS)
+ return Success;
+ switch (Result->Code) {
+ case OL_ERRC_INVALID_VALUE:
+ return ErrorInvalidValue;
+ default:
+ return ErrorInvalidValue;
+ }
+}
Error_t Malloc(void **DevPtr, size_t Size) {
ol_device_handle_t Device = olKGetDefaultDevice();
@@ -145,9 +154,9 @@ Error_t DriverGetVersion(int *Version) {
Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
// TODO: [h15] add remaining pci/mem fields
ol_device_handle_t Device = olKGetDefaultDevice();
- size_t name_size = 0;
- olGetDeviceInfoSize(Device, OL_DEVICE_INFO_NAME, &name_size);
- olGetDeviceInfo(Device, OL_DEVICE_INFO_NAME, name_size, &DeviceProp->name[0]);
+ size_t nameSize = 0;
+ olGetDeviceInfoSize(Device, OL_DEVICE_INFO_NAME, &nameSize);
+ olGetDeviceInfo(Device, OL_DEVICE_INFO_NAME, nameSize, &DeviceProp->name[0]);
olGetDeviceInfo(Device, OL_DEVICE_INFO_GLOBAL_MEM_SIZE, sizeof(size_t),
&DeviceProp->totalGlobalMem);
olGetDeviceInfo(Device, OL_DEVICE_INFO_WARP_SIZE, sizeof(uint32_t),
@@ -158,26 +167,44 @@ Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
return Success;
}
+static Error_t getQueueFromStream(Stream_t Stream, ol_queue_handle_t *Queue) {
+ if (!Stream)
+ return ErrorInvalidValue;
+ *Queue = reinterpret_cast<ol_queue_handle_t>(Stream);
+ return Success;
+}
+
Error_t StreamCreate(Stream_t *Stream) {
- // TODO: [h15] implement
- fprintf(stderr, LANGUAGE_STR "StreamCreate is not implemented yet");
- abort();
+ ol_queue_handle_t Queue;
+ olCreateQueue(olKGetDefaultDevice(), &Queue);
+ *Stream = reinterpret_cast<Stream_t>(Queue);
+ return Success;
}
Error_t StreamCreateWithFlags(Stream_t *Stream, unsigned int Flags) {
- // TODO: [h15] implement
- fprintf(stderr, LANGUAGE_STR "StreamCreateWithFlags is not implemented yet");
- abort();
+ if (Flags == StreamCreateWithFlagsFlags::StreamDefault)
+ return StreamCreate(Stream);
+ if (Flags == StreamCreateWithFlagsFlags::StreamNonBlocking) {
+ // FIXME: [h15] implement non-blocking stream creation
+ return StreamCreate(Stream);
+ }
+ return ErrorInvalidValue;
}
Error_t StreamDestroy(Stream_t Stream) {
- // TODO: [h15] implement
- fprintf(stderr, LANGUAGE_STR "StreamDestroy is not implemented yet");
- abort();
+ ol_queue_handle_t Queue;
+ Error_t Err = getQueueFromStream(Stream, &Queue);
+ if (Err != Success)
+ return Err;
+ ol_result_t Result = olDestroyQueue(Queue);
+ return olKConvertResult(Result);
}
Error_t StreamSynchronize(Stream_t Stream) {
- // TODO: [h15] implement
- fprintf(stderr, LANGUAGE_STR "StreamSynchronize is not implemented yet");
- abort();
+ ol_queue_handle_t Queue;
+ Error_t Err = getQueueFromStream(Stream, &Queue);
+ if (Err != Success)
+ return Err;
+ ol_result_t Result = olSyncQueue(Queue);
+ return olKConvertResult(Result);
}
>From fc8287bdbcad56ba3f6852cdb7267093172a7256 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Mon, 13 Jul 2026 10:21:46 -0700
Subject: [PATCH 15/23] add getDevice, setDevice and DeviceCount
---
.../include/kernel/DefineLanguageNames.inc | 1 +
.../include/kernel/LanguageRuntime.h | 167 ++++++++++++++++++
.../include/kernel/UndefineLanguageNames.inc | 1 +
.../languages/kernel/include/ExportedAPI.h | 6 +
offload/languages/kernel/include/State.h | 2 +-
offload/languages/kernel/src/ExportedAPI.cpp | 28 +++
.../languages/kernel/src/LanguageRuntime.cpp | 18 +-
offload/languages/kernel/src/State.cpp | 5 +-
8 files changed, 222 insertions(+), 6 deletions(-)
diff --git a/offload/languages/include/kernel/DefineLanguageNames.inc b/offload/languages/include/kernel/DefineLanguageNames.inc
index d74703af0f787..3e8b4537f4cc5 100644
--- a/offload/languages/include/kernel/DefineLanguageNames.inc
+++ b/offload/languages/include/kernel/DefineLanguageNames.inc
@@ -29,6 +29,7 @@
#define PeekAtLastError COMBINE(LANGUAGE, PeekAtLastError)
#define GetErrorName COMBINE(LANGUAGE, GetErrorName)
#define GetErrorString COMBINE(LANGUAGE, GetErrorString)
+#define GetDevice COMBINE(LANGUAGE, GetDevice)
#define GetDeviceCount COMBINE(LANGUAGE, GetDeviceCount)
#define SetDevice COMBINE(LANGUAGE, SetDevice)
#define HostAlloc COMBINE(LANGUAGE, HostAlloc)
diff --git a/offload/languages/include/kernel/LanguageRuntime.h b/offload/languages/include/kernel/LanguageRuntime.h
index 9469105142ffb..b6e44a0217a55 100644
--- a/offload/languages/include/kernel/LanguageRuntime.h
+++ b/offload/languages/include/kernel/LanguageRuntime.h
@@ -102,6 +102,8 @@ const char *GetErrorName(Error_t Error);
const char *GetErrorString(Error_t Error);
+Error_t GetDevice(int *DeviceNo);
+
Error_t GetDeviceCount(int *Count);
Error_t SetDevice(int DeviceNo);
@@ -121,6 +123,171 @@ static inline Error_t OccupancyMaxPotentialBlockSizeVariableSMem(
"OccupancyMaxPotentialBlockSizeVariableSMem is not implemented yet");
abort();
}
+// template<typename UnaryFunction, class T>
+// static __inline__ __host__ CUDART_DEVICE cudaError_t cudaOccupancyMaxPotentialBlockSizeVariableSMemWithFlags(
+// int *minGridSize,
+// int *blockSize,
+// T func,
+// UnaryFunction blockSizeToDynamicSMemSize,
+// int blockSizeLimit = 0,
+// unsigned int flags = 0)
+// {
+// Error_t status;
+//
+// // Device and function properties
+// int device;
+// struct cudaFuncAttributes attr;
+//
+// // Limits
+// int maxThreadsPerMultiProcessor;
+// int warpSize;
+// int devMaxThreadsPerBlock;
+// int multiProcessorCount;
+// int funcMaxThreadsPerBlock;
+// int occupancyLimit;
+// int granularity;
+//
+// // Recorded maximum
+// int maxBlockSize = 0;
+// int numBlocks = 0;
+// int maxOccupancy = 0;
+//
+// // Temporary
+// int blockSizeToTryAligned;
+// int blockSizeToTry;
+// int blockSizeLimitAligned;
+// int occupancyInBlocks;
+// int occupancyInThreads;
+// size_t dynamicSMemSize;
+//
+// ///////////////////////////
+// // Check user input
+// ///////////////////////////
+//
+// if (!minGridSize || !blockSize || !func) {
+// return ErrorInvalidValue;
+// }
+//
+// //////////////////////////////////////////////
+// // Obtain device and function properties
+// //////////////////////////////////////////////
+//
+// status = ::GetDevice(&device);
+// if (status != Success) {
+// return status;
+// }
+//
+// status = cudaDeviceGetAttribute(
+// &maxThreadsPerMultiProcessor,
+// cudaDevAttrMaxThreadsPerMultiProcessor,
+// device);
+// if (status != cudaSuccess) {
+// return status;
+// }
+//
+// status = cudaDeviceGetAttribute(
+// &warpSize,
+// cudaDevAttrWarpSize,
+// device);
+// if (status != cudaSuccess) {
+// return status;
+// }
+//
+// status = cudaDeviceGetAttribute(
+// &devMaxThreadsPerBlock,
+// cudaDevAttrMaxThreadsPerBlock,
+// device);
+// if (status != cudaSuccess) {
+// return status;
+// }
+//
+// status = cudaDeviceGetAttribute(
+// &multiProcessorCount,
+// cudaDevAttrMultiProcessorCount,
+// device);
+// if (status != cudaSuccess) {
+// return status;
+// }
+//
+// status = cudaFuncGetAttributes(&attr, func);
+// if (status != cudaSuccess) {
+// return status;
+// }
+//
+// funcMaxThreadsPerBlock = attr.maxThreadsPerBlock;
+//
+// /////////////////////////////////////////////////////////////////////////////////
+// // Try each block size, and pick the block size with maximum occupancy
+// /////////////////////////////////////////////////////////////////////////////////
+//
+// occupancyLimit = maxThreadsPerMultiProcessor;
+// granularity = warpSize;
+//
+// if (blockSizeLimit == 0) {
+// blockSizeLimit = devMaxThreadsPerBlock;
+// }
+//
+// if (devMaxThreadsPerBlock < blockSizeLimit) {
+// blockSizeLimit = devMaxThreadsPerBlock;
+// }
+//
+// if (funcMaxThreadsPerBlock < blockSizeLimit) {
+// blockSizeLimit = funcMaxThreadsPerBlock;
+// }
+//
+// blockSizeLimitAligned = ((blockSizeLimit + (granularity - 1)) / granularity) * granularity;
+//
+// for (blockSizeToTryAligned = blockSizeLimitAligned; blockSizeToTryAligned > 0; blockSizeToTryAligned -= granularity) {
+// // This is needed for the first iteration, because
+// // blockSizeLimitAligned could be greater than blockSizeLimit
+// //
+// if (blockSizeLimit < blockSizeToTryAligned) {
+// blockSizeToTry = blockSizeLimit;
+// } else {
+// blockSizeToTry = blockSizeToTryAligned;
+// }
+//
+// dynamicSMemSize = blockSizeToDynamicSMemSize(blockSizeToTry);
+//
+// status = cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(
+// &occupancyInBlocks,
+// func,
+// blockSizeToTry,
+// dynamicSMemSize,
+// flags);
+//
+// if (status != cudaSuccess) {
+// return status;
+// }
+//
+// occupancyInThreads = blockSizeToTry * occupancyInBlocks;
+//
+// if (occupancyInThreads > maxOccupancy) {
+// maxBlockSize = blockSizeToTry;
+// numBlocks = occupancyInBlocks;
+// maxOccupancy = occupancyInThreads;
+// }
+//
+// // Early out if we have reached the maximum
+// //
+// if (occupancyLimit == maxOccupancy) {
+// break;
+// }
+// }
+//
+// ///////////////////////////
+// // Return best available
+// ///////////////////////////
+//
+// // Suggested min grid size to achieve a full machine launch
+// //
+// *minGridSize = numBlocks * multiProcessorCount;
+// *blockSize = maxBlockSize;
+//
+// return status;
+// }
+
+// }
Error_t StreamCreate(Stream_t *stream);
diff --git a/offload/languages/include/kernel/UndefineLanguageNames.inc b/offload/languages/include/kernel/UndefineLanguageNames.inc
index 19b8eebe3994d..08155f689f722 100644
--- a/offload/languages/include/kernel/UndefineLanguageNames.inc
+++ b/offload/languages/include/kernel/UndefineLanguageNames.inc
@@ -26,6 +26,7 @@
#undef PeekAtLastError
#undef GetErrorName
#undef GetErrorString
+#undef GetDevice
#undef GetDeviceCount
#undef SetDevice
#undef HostAlloc
diff --git a/offload/languages/kernel/include/ExportedAPI.h b/offload/languages/kernel/include/ExportedAPI.h
index 4f70c260c2b22..766a0e8a0a298 100644
--- a/offload/languages/kernel/include/ExportedAPI.h
+++ b/offload/languages/kernel/include/ExportedAPI.h
@@ -17,6 +17,12 @@ extern "C" {
ol_device_handle_t olKGetHostDevice();
+ int olKGetDeviceCount();
+
+ ol_device_handle_t olKGetDevice(int *DeviceNo);
+
+ ol_device_handle_t olKSetDefaultDevice(int DeviceNo);
+
ol_queue_handle_t olKGetDefaultQueue();
CallConfigurationTy *olKGetCallConfiguration();
diff --git a/offload/languages/kernel/include/State.h b/offload/languages/kernel/include/State.h
index 02ad623b7b57d..d107943c35c6b 100644
--- a/offload/languages/kernel/include/State.h
+++ b/offload/languages/kernel/include/State.h
@@ -30,9 +30,9 @@ struct ThreadStateTy {
static ol_queue_handle_t getDefaultQueue();
static ol_device_handle_t getDefaultDevice();
static CallConfigurationTy &getCallConfiguration();
+ void setDefaultDevice(ol_device_handle_t Device);
private:
- void setDefaultDevice(ol_device_handle_t Device);
void createDefaultQueue(ol_device_handle_t Device);
ol_device_handle_t DefaultDevice = nullptr;
diff --git a/offload/languages/kernel/src/ExportedAPI.cpp b/offload/languages/kernel/src/ExportedAPI.cpp
index d3f8847744fbb..e3d75b9bd4a1d 100644
--- a/offload/languages/kernel/src/ExportedAPI.cpp
+++ b/offload/languages/kernel/src/ExportedAPI.cpp
@@ -14,6 +14,7 @@
#include "Types.h"
#include "OffloadAPI.h"
+#include "llvm/ADT/ArrayRef.h"
#include <cstdio>
#include <stdint.h>
@@ -33,6 +34,33 @@ ol_device_handle_t olKGetHostDevice() {
return HostDevice;
}
+int olKGetDeviceCount() {
+ int DeviceCount = StateTy::get().getDevices().size();
+ return DeviceCount;
+}
+
+ol_device_handle_t olKGetDevice(int* DeviceNo) {
+ ol_device_handle_t DefaultDevice = ThreadStateTy::getDefaultDevice();
+ int DeviceCount = StateTy::get().getDevices().size();
+ ArrayRef<ol_device_handle_t> Devices = StateTy::get().getDevices();
+ for (int i = 0; i < DeviceCount; i++){
+ if (Devices[i] == DefaultDevice){
+ *DeviceNo = i;
+ return Devices[i];
+ }
+ }
+ return nullptr;
+}
+
+ol_device_handle_t olKSetDefaultDevice(int DeviceNo) {
+ ArrayRef<ol_device_handle_t> Devices = StateTy::get().getDevices();
+ if (DeviceNo < 0 || DeviceNo >= static_cast<int>(Devices.size()))
+ return nullptr;
+ ol_device_handle_t Device = Devices[DeviceNo];
+ ThreadStateTy::get().setDefaultDevice(Device);
+ return Device;
+}
+
ol_queue_handle_t olKGetDefaultQueue() {
ol_queue_handle_t DefaultQueue = ThreadStateTy::getDefaultQueue();
return DefaultQueue;
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index 1ae4201df2b62..7647c0c0535e4 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -9,6 +9,7 @@
//===----------------------------------------------------------------------===//
#include "LanguageRuntime.h"
+#include <cassert>
#ifndef LANGUAGE
#error This file should be included, or used, with a LANGUAGE macro set.
@@ -118,15 +119,22 @@ const char *GetErrorString(Error_t Error) {
return "";
}
+Error_t GetDevice(int *DeviceNo) {
+ ol_device_handle_t Device = olKGetDevice(DeviceNo);
+ if (!Device)
+ return ErrorInvalidValue;
+ return Success;
+}
+
Error_t GetDeviceCount(int *Count) {
- // TODO:
- *Count = 1;
+ *Count = olKGetDeviceCount();
return Success;
}
Error_t SetDevice(int DeviceNo) {
- // TODO:
- return Success;
+ ol_device_handle_t Device = olKSetDefaultDevice(DeviceNo);
+ assert(Device == olKGetDefaultDevice() && "Set Device is not Default Device");
+ return Device ? Success : ErrorInvalidValue;
}
Error_t HostAlloc(void **Ptr, size_t Size, unsigned int Flags) {
@@ -167,6 +175,8 @@ Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
return Success;
}
+// Error_t GetDeviceAttribute()
+
static Error_t getQueueFromStream(Stream_t Stream, ol_queue_handle_t *Queue) {
if (!Stream)
return ErrorInvalidValue;
diff --git a/offload/languages/kernel/src/State.cpp b/offload/languages/kernel/src/State.cpp
index e6ef90a44c7de..9555c490f294f 100644
--- a/offload/languages/kernel/src/State.cpp
+++ b/offload/languages/kernel/src/State.cpp
@@ -78,7 +78,9 @@ ThreadStateTy &ThreadStateTy::get() {
}
ol_device_handle_t ThreadStateTy::getDefaultDevice() {
- ol_device_handle_t DD = nullptr;
+ ol_device_handle_t DD = ThreadStateTy::get().DefaultDevice;
+ if (DD)
+ return DD;
for (ol_device_handle_t Device : StateTy::get().getDevices()) {
DD = Device;
break;
@@ -91,6 +93,7 @@ ol_device_handle_t ThreadStateTy::getDefaultDevice() {
return DD;
}
+
ol_queue_handle_t ThreadStateTy::getDefaultQueue() {
if (!PerThreadQueue) [[likely]]
return StateTy::get().DefaultQueue;
>From fef659df839420519453b84402e2c23cc455d5c6 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Mon, 13 Jul 2026 14:42:19 -0700
Subject: [PATCH 16/23] add OccupancyMax... stub
---
.../include/kernel/LanguageRuntime.h | 191 ++----------------
.../languages/kernel/src/LanguageRuntime.cpp | 2 +-
2 files changed, 17 insertions(+), 176 deletions(-)
diff --git a/offload/languages/include/kernel/LanguageRuntime.h b/offload/languages/include/kernel/LanguageRuntime.h
index b6e44a0217a55..39fcd01a7d641 100644
--- a/offload/languages/include/kernel/LanguageRuntime.h
+++ b/offload/languages/include/kernel/LanguageRuntime.h
@@ -114,181 +114,6 @@ Error_t DriverGetVersion(int *Version);
Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo);
-template <typename UnaryFunction, class T>
-static inline Error_t OccupancyMaxPotentialBlockSizeVariableSMem(
- int *minGridSize, int *blockSize, T func,
- UnaryFunction blockSizeToDynamicSMemSize, int blockSizeLimit = 0) {
- // TODO: [h15] implement
- fprintf(stderr,
- "OccupancyMaxPotentialBlockSizeVariableSMem is not implemented yet");
- abort();
-}
-// template<typename UnaryFunction, class T>
-// static __inline__ __host__ CUDART_DEVICE cudaError_t cudaOccupancyMaxPotentialBlockSizeVariableSMemWithFlags(
-// int *minGridSize,
-// int *blockSize,
-// T func,
-// UnaryFunction blockSizeToDynamicSMemSize,
-// int blockSizeLimit = 0,
-// unsigned int flags = 0)
-// {
-// Error_t status;
-//
-// // Device and function properties
-// int device;
-// struct cudaFuncAttributes attr;
-//
-// // Limits
-// int maxThreadsPerMultiProcessor;
-// int warpSize;
-// int devMaxThreadsPerBlock;
-// int multiProcessorCount;
-// int funcMaxThreadsPerBlock;
-// int occupancyLimit;
-// int granularity;
-//
-// // Recorded maximum
-// int maxBlockSize = 0;
-// int numBlocks = 0;
-// int maxOccupancy = 0;
-//
-// // Temporary
-// int blockSizeToTryAligned;
-// int blockSizeToTry;
-// int blockSizeLimitAligned;
-// int occupancyInBlocks;
-// int occupancyInThreads;
-// size_t dynamicSMemSize;
-//
-// ///////////////////////////
-// // Check user input
-// ///////////////////////////
-//
-// if (!minGridSize || !blockSize || !func) {
-// return ErrorInvalidValue;
-// }
-//
-// //////////////////////////////////////////////
-// // Obtain device and function properties
-// //////////////////////////////////////////////
-//
-// status = ::GetDevice(&device);
-// if (status != Success) {
-// return status;
-// }
-//
-// status = cudaDeviceGetAttribute(
-// &maxThreadsPerMultiProcessor,
-// cudaDevAttrMaxThreadsPerMultiProcessor,
-// device);
-// if (status != cudaSuccess) {
-// return status;
-// }
-//
-// status = cudaDeviceGetAttribute(
-// &warpSize,
-// cudaDevAttrWarpSize,
-// device);
-// if (status != cudaSuccess) {
-// return status;
-// }
-//
-// status = cudaDeviceGetAttribute(
-// &devMaxThreadsPerBlock,
-// cudaDevAttrMaxThreadsPerBlock,
-// device);
-// if (status != cudaSuccess) {
-// return status;
-// }
-//
-// status = cudaDeviceGetAttribute(
-// &multiProcessorCount,
-// cudaDevAttrMultiProcessorCount,
-// device);
-// if (status != cudaSuccess) {
-// return status;
-// }
-//
-// status = cudaFuncGetAttributes(&attr, func);
-// if (status != cudaSuccess) {
-// return status;
-// }
-//
-// funcMaxThreadsPerBlock = attr.maxThreadsPerBlock;
-//
-// /////////////////////////////////////////////////////////////////////////////////
-// // Try each block size, and pick the block size with maximum occupancy
-// /////////////////////////////////////////////////////////////////////////////////
-//
-// occupancyLimit = maxThreadsPerMultiProcessor;
-// granularity = warpSize;
-//
-// if (blockSizeLimit == 0) {
-// blockSizeLimit = devMaxThreadsPerBlock;
-// }
-//
-// if (devMaxThreadsPerBlock < blockSizeLimit) {
-// blockSizeLimit = devMaxThreadsPerBlock;
-// }
-//
-// if (funcMaxThreadsPerBlock < blockSizeLimit) {
-// blockSizeLimit = funcMaxThreadsPerBlock;
-// }
-//
-// blockSizeLimitAligned = ((blockSizeLimit + (granularity - 1)) / granularity) * granularity;
-//
-// for (blockSizeToTryAligned = blockSizeLimitAligned; blockSizeToTryAligned > 0; blockSizeToTryAligned -= granularity) {
-// // This is needed for the first iteration, because
-// // blockSizeLimitAligned could be greater than blockSizeLimit
-// //
-// if (blockSizeLimit < blockSizeToTryAligned) {
-// blockSizeToTry = blockSizeLimit;
-// } else {
-// blockSizeToTry = blockSizeToTryAligned;
-// }
-//
-// dynamicSMemSize = blockSizeToDynamicSMemSize(blockSizeToTry);
-//
-// status = cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(
-// &occupancyInBlocks,
-// func,
-// blockSizeToTry,
-// dynamicSMemSize,
-// flags);
-//
-// if (status != cudaSuccess) {
-// return status;
-// }
-//
-// occupancyInThreads = blockSizeToTry * occupancyInBlocks;
-//
-// if (occupancyInThreads > maxOccupancy) {
-// maxBlockSize = blockSizeToTry;
-// numBlocks = occupancyInBlocks;
-// maxOccupancy = occupancyInThreads;
-// }
-//
-// // Early out if we have reached the maximum
-// //
-// if (occupancyLimit == maxOccupancy) {
-// break;
-// }
-// }
-//
-// ///////////////////////////
-// // Return best available
-// ///////////////////////////
-//
-// // Suggested min grid size to achieve a full machine launch
-// //
-// *minGridSize = numBlocks * multiProcessorCount;
-// *blockSize = maxBlockSize;
-//
-// return status;
-// }
-
-// }
-
Error_t StreamCreate(Stream_t *stream);
Error_t StreamCreateWithFlags(Stream_t *stream, unsigned int flags);
@@ -297,6 +122,22 @@ Error_t StreamDestroy(Stream_t stream);
Error_t StreamSynchronize(Stream_t stream);
+template <typename UnaryFunction, class T>
+static inline Error_t OccupancyMaxPotentialBlockSizeVariableSMem(
+ int *minGridSize, int *blockSize, T func,
+ UnaryFunction blockSizeToDynamicSMemSize, int blockSizeLimit = 0) {
+#if defined(__AMDGPU__)
+ // TODO: values taken from AMD Instinct MI250X gfx90a
+ *minGridSize = 220;
+ *blockSize = 1024;
+#elif defined(__NVPTX__)
+ // TODO: values taken from NVIDIA H100 80GB HBM3
+ *minGridSize = 264;
+ *blockSize = 1024;
+#endif
+ return Success;
+}
+
///
#if defined(__AMDGPU__) || defined(__NVPTX__)
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index 7647c0c0535e4..3e9cfe371379f 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -193,9 +193,9 @@ Error_t StreamCreate(Stream_t *Stream) {
Error_t StreamCreateWithFlags(Stream_t *Stream, unsigned int Flags) {
if (Flags == StreamCreateWithFlagsFlags::StreamDefault)
+ // FIXME: [h15] offload streams are non-blocking by default
return StreamCreate(Stream);
if (Flags == StreamCreateWithFlagsFlags::StreamNonBlocking) {
- // FIXME: [h15] implement non-blocking stream creation
return StreamCreate(Stream);
}
return ErrorInvalidValue;
>From c275ae7c88eec66d5f4b19a8277c256e7fe50a15 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Mon, 13 Jul 2026 18:17:30 -0700
Subject: [PATCH 17/23] add better kernel cleanup
---
.../languages/kernel/include/ExportedAPI.h | 26 ++++++----
offload/languages/kernel/include/State.h | 14 ++++++
offload/languages/kernel/src/ExportedAPI.cpp | 19 +++++--
.../kernel/src/LanguageRegistration.cpp | 22 ++++++--
offload/languages/kernel/src/State.cpp | 50 +++++++++++++++----
5 files changed, 102 insertions(+), 29 deletions(-)
diff --git a/offload/languages/kernel/include/ExportedAPI.h b/offload/languages/kernel/include/ExportedAPI.h
index 766a0e8a0a298..a4ce168d78fda 100644
--- a/offload/languages/kernel/include/ExportedAPI.h
+++ b/offload/languages/kernel/include/ExportedAPI.h
@@ -13,25 +13,29 @@
#include "Types.h"
extern "C" {
- ol_device_handle_t olKGetDefaultDevice();
+ol_device_handle_t olKGetDefaultDevice();
- ol_device_handle_t olKGetHostDevice();
+ol_device_handle_t olKGetHostDevice();
- int olKGetDeviceCount();
+int olKGetDeviceCount();
- ol_device_handle_t olKGetDevice(int *DeviceNo);
+ol_device_handle_t olKGetDevice(int *DeviceNo);
- ol_device_handle_t olKSetDefaultDevice(int DeviceNo);
+ol_device_handle_t olKSetDefaultDevice(int DeviceNo);
- ol_queue_handle_t olKGetDefaultQueue();
+ol_queue_handle_t olKGetDefaultQueue();
- CallConfigurationTy *olKGetCallConfiguration();
+CallConfigurationTy *olKGetCallConfiguration();
- void olKRegisterKernel(const void *ID, ol_symbol_handle_t Kernel);
+void olKRegisterKernel(const void *ID, ol_symbol_handle_t Kernel);
- ol_symbol_handle_t olKGetKernel(const void *ID);
+void olKUnregisterKernel(const void *ID);
- void olKRegisterProgram(const void *ID, ol_program_handle_t Program);
+ol_symbol_handle_t olKGetKernel(const void *ID);
- ol_program_handle_t olKGetProgram(const void *ID);
+void olKRegisterProgram(const void *ID, ol_program_handle_t Program);
+
+ol_program_handle_t olKUnregisterProgram(const void *ID);
+
+ol_program_handle_t olKGetProgram(const void *ID);
}
diff --git a/offload/languages/kernel/include/State.h b/offload/languages/kernel/include/State.h
index d107943c35c6b..90f323d160a04 100644
--- a/offload/languages/kernel/include/State.h
+++ b/offload/languages/kernel/include/State.h
@@ -49,6 +49,7 @@ struct StateTy {
friend struct ThreadStateTy;
static StateTy &get();
+ static StateTy *tryGet();
static ol_device_handle_t getHostDevice() { return get().HostDevice; }
@@ -64,6 +65,8 @@ struct StateTy {
KernelMap[KernelID] = Kernel;
}
+ void removeKernel(KernelIDTy KernelID) { KernelMap.erase(KernelID); }
+
ol_symbol_handle_t getKernel(KernelIDTy KernelID) {
return KernelMap[KernelID];
}
@@ -72,11 +75,22 @@ struct StateTy {
BinaryRegisterMap[Binary] = Program;
}
+ ol_program_handle_t removeProgram(const void *Binary) {
+ auto It = BinaryRegisterMap.find(Binary);
+ if (It == BinaryRegisterMap.end())
+ return nullptr;
+ ol_program_handle_t Program = It->second;
+ BinaryRegisterMap.erase(It);
+ return Program;
+ }
+
ol_program_handle_t getProgram(const void *Binary) {
assert(BinaryRegisterMap.count(Binary));
return BinaryRegisterMap[Binary];
}
+ void destroyRegisteredPrograms();
+
private:
DenseMap<const void *, ol_program_handle_t> BinaryRegisterMap;
DenseMap<KernelIDTy, ol_symbol_handle_t> KernelMap;
diff --git a/offload/languages/kernel/src/ExportedAPI.cpp b/offload/languages/kernel/src/ExportedAPI.cpp
index e3d75b9bd4a1d..5c65d3d66b89b 100644
--- a/offload/languages/kernel/src/ExportedAPI.cpp
+++ b/offload/languages/kernel/src/ExportedAPI.cpp
@@ -39,12 +39,12 @@ int olKGetDeviceCount() {
return DeviceCount;
}
-ol_device_handle_t olKGetDevice(int* DeviceNo) {
+ol_device_handle_t olKGetDevice(int *DeviceNo) {
ol_device_handle_t DefaultDevice = ThreadStateTy::getDefaultDevice();
int DeviceCount = StateTy::get().getDevices().size();
ArrayRef<ol_device_handle_t> Devices = StateTy::get().getDevices();
- for (int i = 0; i < DeviceCount; i++){
- if (Devices[i] == DefaultDevice){
+ for (int i = 0; i < DeviceCount; i++) {
+ if (Devices[i] == DefaultDevice) {
*DeviceNo = i;
return Devices[i];
}
@@ -66,7 +66,7 @@ ol_queue_handle_t olKGetDefaultQueue() {
return DefaultQueue;
}
-CallConfigurationTy* olKGetCallConfiguration() {
+CallConfigurationTy *olKGetCallConfiguration() {
return &ThreadStateTy::getCallConfiguration();
}
@@ -74,6 +74,11 @@ void olKRegisterKernel(const void *ID, ol_symbol_handle_t Kernel) {
StateTy::get().addKernel(ID, Kernel);
}
+void olKUnregisterKernel(const void *ID) {
+ if (StateTy *State = StateTy::tryGet())
+ State->removeKernel(ID);
+}
+
ol_symbol_handle_t olKGetKernel(const void *ID) {
return StateTy::get().getKernel(ID);
}
@@ -82,6 +87,12 @@ void olKRegisterProgram(const void *ID, ol_program_handle_t Program) {
StateTy::get().addProgram(ID, Program);
}
+ol_program_handle_t olKUnregisterProgram(const void *ID) {
+ if (StateTy *State = StateTy::tryGet())
+ return State->removeProgram(ID);
+ return nullptr;
+}
+
ol_program_handle_t olKGetProgram(const void *ID) {
return StateTy::get().getProgram(ID);
}
diff --git a/offload/languages/kernel/src/LanguageRegistration.cpp b/offload/languages/kernel/src/LanguageRegistration.cpp
index 6171e6a92afd6..1be65cd63aa34 100644
--- a/offload/languages/kernel/src/LanguageRegistration.cpp
+++ b/offload/languages/kernel/src/LanguageRegistration.cpp
@@ -233,7 +233,10 @@ const char *__llvmRegisterFatBinary(const char *Binary) {
return Binary;
}
-void __llvmUnregisterFatBinary(void *Handle) {}
+void __llvmUnregisterFatBinary(void *Handle) {
+ if (ol_program_handle_t Program = olKUnregisterProgram(Handle))
+ olDestroyProgram(Program);
+}
void __llvmRegisterVar(void **, char *, char *, const char *, int, int, int,
int) {
@@ -300,13 +303,26 @@ void __tgt_register_lib(__tgt_bin_desc *Desc) {
Entry != DeviceImage.EntriesEnd; ++Entry) {
if (!Entry->Size && !Entry->Flags)
__llvmRegisterFunction((const char *)DeviceImage.ImageStart,
- (const char*)Entry->Address, Entry->SymbolName,
+ (const char *)Entry->Address, Entry->SymbolName,
Entry->SymbolName, 0, nullptr, nullptr, nullptr,
nullptr, nullptr);
}
}
}
-void __tgt_unregister_lib(__tgt_bin_desc *Desc) {}
+void __tgt_unregister_lib(__tgt_bin_desc *Desc) {
+ for (int32_t I = 0, E = Desc->NumDeviceImages; I < E; ++I) {
+ __tgt_device_image &DeviceImage = Desc->DeviceImages[I];
+ for (auto *Entry = DeviceImage.EntriesBegin;
+ Entry != DeviceImage.EntriesEnd; ++Entry) {
+ if (!Entry->Size && !Entry->Flags)
+ olKUnregisterKernel((const char *)Entry->Address);
+ }
+
+ if (ol_program_handle_t Program =
+ olKUnregisterProgram(DeviceImage.ImageStart))
+ olDestroyProgram(Program);
+ }
+}
}
///}
diff --git a/offload/languages/kernel/src/State.cpp b/offload/languages/kernel/src/State.cpp
index 9555c490f294f..eff3c31257e93 100644
--- a/offload/languages/kernel/src/State.cpp
+++ b/offload/languages/kernel/src/State.cpp
@@ -12,6 +12,7 @@
#include "Types.h"
#include "OffloadAPI.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Support/raw_ostream.h"
#include <atomic>
@@ -43,9 +44,15 @@ static ThreadStatesTy *ThreadStatesPtr = nullptr;
static void deleteThreadState() {
std::lock_guard<std::mutex> LG(ThreadStatesLock);
- if (ThreadStatesPtr)
- for (auto *TS : *ThreadStatesPtr)
- delete (TS);
+ ThreadStatesTy *ThreadStates = ThreadStatesPtr;
+ ThreadStatesPtr = nullptr;
+ if (!ThreadStates)
+ return;
+
+ for (auto *TS : *ThreadStates)
+ delete (TS);
+ delete ThreadStates;
+ ThreadState = nullptr;
}
static void deleteState() {
@@ -54,15 +61,21 @@ static void deleteState() {
delete (ST);
}
+static void destroyQueue(ol_queue_handle_t &Queue) {
+ if (!Queue)
+ return;
+
+ olSyncQueue(Queue);
+ olDestroyQueue(Queue);
+ Queue = nullptr;
+}
+
ThreadStateTy::ThreadStateTy() {
if (PerThreadQueue) [[unlikely]]
createDefaultQueue(getDefaultDevice());
atexit(deleteThreadState);
}
-ThreadStateTy::~ThreadStateTy() {
- if (DefaultQueue)
- olSyncQueue(DefaultQueue);
-}
+ThreadStateTy::~ThreadStateTy() { destroyQueue(DefaultQueue); }
ThreadStateTy &ThreadStateTy::get() {
auto *TS = ThreadState;
@@ -93,14 +106,13 @@ ol_device_handle_t ThreadStateTy::getDefaultDevice() {
return DD;
}
-
ol_queue_handle_t ThreadStateTy::getDefaultQueue() {
if (!PerThreadQueue) [[likely]]
return StateTy::get().DefaultQueue;
return ThreadStateTy::get().DefaultQueue;
}
-CallConfigurationTy& ThreadStateTy::getCallConfiguration() {
+CallConfigurationTy &ThreadStateTy::getCallConfiguration() {
return ThreadStateTy::get().CC;
}
@@ -129,6 +141,8 @@ StateTy &StateTy::get() {
return *ST;
}
+StateTy *StateTy::tryGet() { return StatePtr.load(); }
+
static bool addDevices(ol_device_handle_t Device, void *Payload) {
StateTy &State = *reinterpret_cast<StateTy *>(Payload);
ol_platform_handle_t Platform;
@@ -165,6 +179,20 @@ StateTy::StateTy() {
}
StateTy::~StateTy() {
- if (DefaultQueue)
- olSyncQueue(DefaultQueue);
+ deleteThreadState();
+ destroyQueue(DefaultQueue);
+ destroyRegisteredPrograms();
+ olShutDown();
+}
+
+void StateTy::destroyRegisteredPrograms() {
+ SmallPtrSet<ol_program_handle_t, 8> Programs;
+ for (auto &It : BinaryRegisterMap)
+ Programs.insert(It.second);
+
+ KernelMap.clear();
+ BinaryRegisterMap.clear();
+
+ for (ol_program_handle_t Program : Programs)
+ olDestroyProgram(Program);
}
>From 2769cef97e4a422ff893eaf9366c3cfae7d75bc6 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Wed, 15 Jul 2026 11:12:34 -0700
Subject: [PATCH 18/23] add hip offloading on AMD
---
clang/lib/Driver/ToolChains/AMDGPU.cpp | 19 +++++++++++++++++++
clang/lib/Driver/ToolChains/Clang.cpp | 19 +++++++++++++++++--
clang/lib/Driver/ToolChains/Cuda.cpp | 3 +--
clang/lib/Driver/ToolChains/HIPAMD.cpp | 14 ++++++++++++--
clang/lib/Driver/ToolChains/Linux.cpp | 4 +++-
offload/languages/include/hip/hip_runtime.h | 2 +-
.../languages/kernel/src/LanguageRuntime.cpp | 5 ++---
7 files changed, 55 insertions(+), 11 deletions(-)
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 032c108d3d10d..8de997b78cb84 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -514,6 +514,25 @@ void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
!DriverArgs.hasArg(options::OPT_nohipwrapperinc);
bool HasHipStdPar = DriverArgs.hasArg(options::OPT_hipstdpar);
+ // if (D.getTargetTriple() == llvm::Triple::LLVM &&
+ if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false)) {
+ if (DriverArgs.hasFlag(options::OPT_offload_inc,
+ options::OPT_no_offload_inc, true) &&
+ !DriverArgs.hasArg(options::OPT_nohipwrapperinc) &&
+ !DriverArgs.hasArg(options::OPT_nobuiltininc)) {
+ CC1Args.append({"-include", "__clang_gpu_device_functions.h"});
+
+ SmallString<128> HIPIncludePath(D.ResourceDir);
+ llvm::sys::path::append(HIPIncludePath, "..", "..", "..");
+ llvm::sys::path::append(HIPIncludePath, "include", "offload", "hip");
+ CC1Args.push_back("-internal-isystem");
+ CC1Args.push_back(DriverArgs.MakeArgString(HIPIncludePath));
+ CC1Args.append({"-include", "hip_runtime.h"});
+ }
+ return;
+ }
+
if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
// HIP header includes standard library wrapper headers under clang
// cuda_wrappers directory. Since these wrapper headers include_next
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 32786dbd60fe2..552bed5a577ee 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -962,7 +962,7 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
getToolChain().printVerboseInfo(llvm::errs());
getToolChain().AddCudaIncludeArgs(Args, CmdArgs);
}
- if (JA.isOffloading(Action::OFK_HIP))
+ if (JA.isOffloading(Action::OFK_HIP) && !UsesLLVMOffloading)
getToolChain().AddHIPIncludeArgs(Args, CmdArgs);
if (JA.isOffloading(Action::OFK_SYCL))
getToolChain().addSYCLIncludeArgs(Args, CmdArgs);
@@ -992,7 +992,6 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
if (UsesLLVMOffloading && JA.isOffloading(Action::OFK_Cuda) &&
Args.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
true) &&
- !Args.hasArg(options::OPT_nohipwrapperinc) &&
!Args.hasArg(options::OPT_nobuiltininc) && isCudaInput) {
CmdArgs.append({"-include", "__clang_gpu_device_functions.h"});
@@ -1003,6 +1002,22 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
"-include"});
CmdArgs.push_back("cuda_runtime.h");
}
+ bool isHIPInput = llvm::any_of(
+ Inputs, [](const InputInfo &I) { return types::isHIP(I.getType()); });
+ if (UsesLLVMOffloading && JA.isOffloading(Action::OFK_HIP) &&
+ Args.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
+ true) &&
+ !Args.hasArg(options::OPT_nohipwrapperinc) &&
+ !Args.hasArg(options::OPT_nobuiltininc) && isHIPInput) {
+ CmdArgs.append({"-include", "__clang_gpu_device_functions.h"});
+
+ SmallString<128> OffloadHIPInclude(D.Dir);
+ llvm::sys::path::append(OffloadHIPInclude, "..", "include", "offload",
+ "hip");
+ CmdArgs.append({"-internal-isystem", Args.MakeArgString(OffloadHIPInclude),
+ "-include"});
+ CmdArgs.push_back("hip_runtime.h");
+ }
// Add -i* options, and automatically translate to
// -include-pch/-include-pth for transparent PCH support. It's
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 0a91c411d528c..b60e351178855 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -306,8 +306,7 @@ void CudaInstallationDetector::AddCudaIncludeArgs(
options::OPT_fno_offload_via_llvm, false)) {
if (DriverArgs.hasFlag(options::OPT_offload_inc,
options::OPT_no_offload_inc, true) &&
- !DriverArgs.hasArg(options::OPT_nohipwrapperinc) &&
- !DriverArgs.hasArg(options::OPT_nobuiltininc)){
+ !DriverArgs.hasArg(options::OPT_nobuiltininc)) {
CC1Args.append({"-include", "__clang_gpu_device_functions.h"});
SmallString<128> CudaIncludePath(D.ResourceDir);
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index abcba23a370fe..9d81a6a68128a 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -325,12 +325,22 @@ void HIPAMDToolChain::AddIAMCUIncludeArgs(const ArgList &Args,
void HIPAMDToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
ArgStringList &CC1Args) const {
- if (getTriple().getEnvironment() == llvm::Triple::LLVM) {
+ if (getTriple().getEnvironment() == llvm::Triple::LLVM &&
+ DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false)) {
if (DriverArgs.hasFlag(options::OPT_offload_inc,
options::OPT_no_offload_inc, true) &&
!DriverArgs.hasArg(options::OPT_nohipwrapperinc) &&
- !DriverArgs.hasArg(options::OPT_nobuiltininc))
+ !DriverArgs.hasArg(options::OPT_nobuiltininc)) {
CC1Args.append({"-include", "__clang_gpu_device_functions.h"});
+
+ SmallString<128> HIPIncludePath(getDriver().ResourceDir);
+ llvm::sys::path::append(HIPIncludePath, "..", "..", "..");
+ llvm::sys::path::append(HIPIncludePath, "include", "offload", "hip");
+ CC1Args.push_back("-internal-isystem");
+ CC1Args.push_back(DriverArgs.MakeArgString(HIPIncludePath));
+ CC1Args.append({"-include", "hip_runtime.h"});
+ }
return;
}
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 8d30a5ead3bbb..6f7876beca8c1 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -884,7 +884,9 @@ void Linux::addOffloadRTLibs(unsigned ActiveKinds, const ArgList &Args,
if (!Args.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
true) ||
Args.hasArg(options::OPT_nostdlib) ||
- Args.hasArg(options::OPT_no_hip_rt) || Args.hasArg(options::OPT_r))
+ Args.hasArg(options::OPT_no_hip_rt) || Args.hasArg(options::OPT_r) ||
+ Args.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false))
return;
llvm::SmallVector<std::pair<StringRef, StringRef>> Libraries;
diff --git a/offload/languages/include/hip/hip_runtime.h b/offload/languages/include/hip/hip_runtime.h
index fd0f31492420a..44fec6cf1933f 100644
--- a/offload/languages/include/hip/hip_runtime.h
+++ b/offload/languages/include/hip/hip_runtime.h
@@ -27,7 +27,7 @@ enum hipHostMallocFlag_t : unsigned int {
hipHostMallocNonCoherent = 0x80000000,
};
-hipError_t hipHostMalloc(void **Ptr, size_t Size, unsigned int Flags) {
+inline hipError_t hipHostMalloc(void **Ptr, size_t Size, unsigned int Flags) {
return hipHostAlloc(Ptr, Size, Flags);
}
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index 3e9cfe371379f..43c6e0b994124 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -167,11 +167,10 @@ Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
olGetDeviceInfo(Device, OL_DEVICE_INFO_NAME, nameSize, &DeviceProp->name[0]);
olGetDeviceInfo(Device, OL_DEVICE_INFO_GLOBAL_MEM_SIZE, sizeof(size_t),
&DeviceProp->totalGlobalMem);
+ olGetDeviceInfo(Device, OL_DEVICE_INFO_NUM_COMPUTE_UNITS, sizeof(uint32_t),
+ &DeviceProp->multiProcessorCount);
olGetDeviceInfo(Device, OL_DEVICE_INFO_WARP_SIZE, sizeof(uint32_t),
&DeviceProp->warpSize);
- DeviceProp->multiProcessorCount = 110;
- DeviceProp->major = 47;
- DeviceProp->minor = 11;
return Success;
}
>From 545799834c6f25c5496b7fc07e486e216af7e1ed Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Wed, 15 Jul 2026 13:18:05 -0700
Subject: [PATCH 19/23] add hip offloading on nvidia
---
clang/lib/Driver/Driver.cpp | 1 -
clang/lib/Driver/ToolChains/Cuda.cpp | 15 ++++++---------
2 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index b24e21c524c04..c92d475f3b2bc 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -7005,7 +7005,6 @@ const ToolChain &Driver::getOffloadToolChain(
Args);
break;
case llvm::Triple::AMDHSA:
- llvm::errs() << "raa did indeed make it into here\n";
if (Kind == Action::OFK_HIP)
TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
*HostTC, Args);
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index b60e351178855..e2df83b21d76a 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -915,9 +915,14 @@ void CudaToolChain::addClangTargetOptions(
BoundArch BA, Action::OffloadKind DeviceOffloadingKind) const {
HostTC.addClangTargetOptions(DriverArgs, CC1Args, BA, DeviceOffloadingKind);
+ bool UseLLVMOffload =
+ getTriple().getEnvironment() == llvm::Triple::LLVM &&
+ DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+
StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);
assert((DeviceOffloadingKind == Action::OFK_OpenMP ||
- DeviceOffloadingKind == Action::OFK_Cuda) &&
+ DeviceOffloadingKind == Action::OFK_Cuda || UseLLVMOffload) &&
"Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs.");
CC1Args.append({"-fcuda-is-device", "-mllvm",
@@ -936,14 +941,6 @@ void CudaToolChain::addClangTargetOptions(
DriverArgs.hasArg(options::OPT_S))
return;
- // For now, we don't use any Offload/OpenMP device runtime when we offload
- // CUDA via LLVM/Offload. We should split the Offload/OpenMP device runtime
- // and include the "generic" (or CUDA-specific) parts.
- // When using LLVM offload path, we use __clang_gpu_device_functions.h
- // instead of libdevice.
- bool UseLLVMOffload = getTriple().getEnvironment() == llvm::Triple::LLVM ||
- DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
if (UseLLVMOffload)
return;
>From db91505c36276c1fc40a3b13df6fe1ed995de0d4 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 16 Jul 2026 09:30:43 -0700
Subject: [PATCH 20/23] fix hip_runtime.h include
---
clang/lib/Driver/ToolChains/AMDGPU.cpp | 5 ++---
clang/lib/Driver/ToolChains/Clang.cpp | 7 +++----
clang/lib/Driver/ToolChains/HIPAMD.cpp | 4 ++--
offload/languages/include/hip/hip_runtime.h | 7 +++++++
4 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 8de997b78cb84..7d2a9d8cc5a28 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -514,7 +514,6 @@ void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
!DriverArgs.hasArg(options::OPT_nohipwrapperinc);
bool HasHipStdPar = DriverArgs.hasArg(options::OPT_hipstdpar);
- // if (D.getTargetTriple() == llvm::Triple::LLVM &&
if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
options::OPT_fno_offload_via_llvm, false)) {
if (DriverArgs.hasFlag(options::OPT_offload_inc,
@@ -525,10 +524,10 @@ void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
SmallString<128> HIPIncludePath(D.ResourceDir);
llvm::sys::path::append(HIPIncludePath, "..", "..", "..");
- llvm::sys::path::append(HIPIncludePath, "include", "offload", "hip");
+ llvm::sys::path::append(HIPIncludePath, "include", "offload");
CC1Args.push_back("-internal-isystem");
CC1Args.push_back(DriverArgs.MakeArgString(HIPIncludePath));
- CC1Args.append({"-include", "hip_runtime.h"});
+ CC1Args.append({"-include", "hip/hip_runtime.h"});
}
return;
}
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 552bed5a577ee..3a183ea37513a 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1012,11 +1012,10 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
CmdArgs.append({"-include", "__clang_gpu_device_functions.h"});
SmallString<128> OffloadHIPInclude(D.Dir);
- llvm::sys::path::append(OffloadHIPInclude, "..", "include", "offload",
- "hip");
+ llvm::sys::path::append(OffloadHIPInclude, "..", "include", "offload");
CmdArgs.append({"-internal-isystem", Args.MakeArgString(OffloadHIPInclude),
"-include"});
- CmdArgs.push_back("hip_runtime.h");
+ CmdArgs.push_back("hip/hip_runtime.h");
}
// Add -i* options, and automatically translate to
@@ -5189,7 +5188,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
bool IsOpenMPDevice = JA.isDeviceOffloading(Action::OFK_OpenMP);
bool IsExtractAPI = isa<ExtractAPIJobAction>(JA);
bool UsesLLVMOffloading =
- Triple.getEnvironment() == llvm::Triple::LLVM ||
+ Triple.getEnvironment() == llvm::Triple::LLVM &&
Args.hasFlag(options::OPT_foffload_via_llvm,
options::OPT_fno_offload_via_llvm, false);
bool IsDeviceOffloadAction = !(JA.isDeviceOffloading(Action::OFK_None) ||
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index 9d81a6a68128a..b1d09ffef919b 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -336,10 +336,10 @@ void HIPAMDToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
SmallString<128> HIPIncludePath(getDriver().ResourceDir);
llvm::sys::path::append(HIPIncludePath, "..", "..", "..");
- llvm::sys::path::append(HIPIncludePath, "include", "offload", "hip");
+ llvm::sys::path::append(HIPIncludePath, "include", "offload");
CC1Args.push_back("-internal-isystem");
CC1Args.push_back(DriverArgs.MakeArgString(HIPIncludePath));
- CC1Args.append({"-include", "hip_runtime.h"});
+ CC1Args.append({"-include", "hip/hip_runtime.h"});
}
return;
}
diff --git a/offload/languages/include/hip/hip_runtime.h b/offload/languages/include/hip/hip_runtime.h
index 44fec6cf1933f..3d113b5945fcc 100644
--- a/offload/languages/include/hip/hip_runtime.h
+++ b/offload/languages/include/hip/hip_runtime.h
@@ -31,12 +31,19 @@ inline hipError_t hipHostMalloc(void **Ptr, size_t Size, unsigned int Flags) {
return hipHostAlloc(Ptr, Size, Flags);
}
+inline hipError_t hipHostFree(void *Ptr) { return ::hipFreeHost(Ptr); }
+
template <class T>
static inline hipError_t hipHostMalloc(T **Ptr, size_t Size,
unsigned int Flags) {
return ::hipHostMalloc((void **)Ptr, Size, Flags);
}
+template <class T>
+static inline hipError_t hipHostFree(T *Ptr) {
+ return ::hipHostFree((void *)Ptr);
+}
+
#if defined(__AMDGPU__) || defined(__NVPTX__)
#define HIP_KERNEL_NAME(...) __VA_ARGS__
>From 4eb4d30b90761420b099686cf37ccff9bed33e3a Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 16 Jul 2026 10:30:47 -0700
Subject: [PATCH 21/23] switch to depend only on -foffliad-via-llvm
---
clang/lib/CodeGen/CGCUDANV.cpp | 7 +--
clang/lib/Driver/Driver.cpp | 61 +++++++++++++-------------
clang/lib/Driver/ToolChains/Clang.cpp | 16 +++----
clang/lib/Driver/ToolChains/Cuda.cpp | 29 +++++-------
clang/lib/Driver/ToolChains/HIPAMD.cpp | 9 ++--
5 files changed, 58 insertions(+), 64 deletions(-)
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 7e2022643ab22..5c1107f4a27dd 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -412,9 +412,9 @@ Address CGNVCUDARuntime::prepareKernelArgs(CodeGenFunction &CGF,
// array and kernels are launched using cudaLaunchKernel().
void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
FunctionArgList &Args) {
- bool UseLLVMOffload = CGF.getLangOpts().OffloadViaLLVM;
+ bool UsesLLVMOffloading = CGF.getLangOpts().OffloadViaLLVM;
// Build the shadow stack entry at the very start of the function.
- Address KernelArgs = UseLLVMOffload
+ Address KernelArgs = UsesLLVMOffloading
? prepareKernelArgsLLVMOffload(CGF, Args) :
prepareKernelArgs(CGF, Args);
@@ -441,7 +441,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
KernelLaunchAPI = KernelLaunchAPI + "_ptsz";
}
/// Use __llvmLaunchKernel for LLVMOffload.
- auto LaunchKernelName = UseLLVMOffload ? "__llvm" + KernelLaunchAPI : addPrefixToName(KernelLaunchAPI);
+ auto LaunchKernelName = UsesLLVMOffloading ? "__llvm" + KernelLaunchAPI
+ : addPrefixToName(KernelLaunchAPI);
const IdentifierInfo &cudaLaunchKernelII =
CGM.getContext().Idents.get(LaunchKernelName);
FunctionDecl *cudaLaunchKernelFD = nullptr;
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index c92d475f3b2bc..2b26147d158a1 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -884,10 +884,15 @@ getSystemOffloadArchs(Compilation &C, Action::OffloadKind Kind) {
if (llvm::ErrorOr<std::string> Executable =
llvm::sys::findProgramByName(Program, {C.getDriver().Dir})) {
llvm::SmallVector<StringRef> Args{*Executable};
- if (Kind == Action::OFK_HIP)
- Args.push_back("--only=amdgpu");
- else if (Kind == Action::OFK_Cuda)
- Args.push_back("--only=nvptx");
+ bool UsesLLVMOffloading =
+ C.getArgs().hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+ if (!UsesLLVMOffloading) {
+ if (Kind == Action::OFK_HIP)
+ Args.push_back("--only=amdgpu");
+ else if (Kind == Action::OFK_Cuda)
+ Args.push_back("--only=nvptx");
+ }
auto StdoutOrErr = C.getDriver().executeProgram(Args);
if (!StdoutOrErr) {
@@ -923,10 +928,8 @@ static TripleSet inferOffloadToolchains(Compilation &C,
for (StringRef Arch : A->getValues()) {
if (A->getOption().matches(options::OPT_offload_arch_EQ)) {
if (Arch == "native") {
- for (StringRef Str : getSystemOffloadArchs(C, Kind)) {
- llvm::errs() << "Inserting into Archs: " << Str.str() << '\n';
+ for (StringRef Str : getSystemOffloadArchs(C, Kind))
Archs.insert(Str.str());
- }
} else {
Archs.insert(Arch.str());
}
@@ -947,16 +950,20 @@ static TripleSet inferOffloadToolchains(Compilation &C,
llvm::Triple(llvm::Triple::amdgcn, llvm::Triple::NoSubArch,
llvm::Triple::AMD, llvm::Triple::AMDHSA),
Arch));
-
- if (Kind == Action::OFK_HIP && !IsAMDOffloadArch(ID)) {
- C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
- << "HIP" << Arch;
- return {};
- }
- if (Kind == Action::OFK_Cuda && !IsNVIDIAOffloadArch(ID)) {
- C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
- << "CUDA" << Arch;
- return {};
+ bool UsesLLVMOffloading =
+ C.getArgs().hasFlag(options::OPT_foffload_via_llvm,
+ options::OPT_fno_offload_via_llvm, false);
+ if (!UsesLLVMOffloading) {
+ if (Kind == Action::OFK_HIP && !IsAMDOffloadArch(ID)) {
+ C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
+ << "HIP" << Arch;
+ return {};
+ }
+ if (Kind == Action::OFK_Cuda && !IsNVIDIAOffloadArch(ID)) {
+ C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
+ << "CUDA" << Arch;
+ return {};
+ }
}
if (Kind == Action::OFK_OpenMP &&
(ID == OffloadArch::Unknown || ID == OffloadArch::Unused)) {
@@ -972,7 +979,8 @@ static TripleSet inferOffloadToolchains(Compilation &C,
llvm::Triple Triple =
OffloadArchToTriple(C.getDefaultToolChain().getTriple(), ID);
- llvm::errs() << "Detected triple: " << Triple.str() << "\n";
+ if (UsesLLVMOffloading)
+ Triple.setEnvironment(llvm::Triple::LLVM);
// Make a new argument that dispatches this argument to the appropriate
// toolchain. This is required when we infer it and create potentially
@@ -1037,12 +1045,13 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
(C.getInputArgs().hasArg(options::OPT_offload_arch_EQ) &&
!(IsCuda || IsHIP))));
- bool UseLLVMOffload = C.getInputArgs().hasArg(
+ bool UsesLLVMOffloading = C.getInputArgs().hasArg(
options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
// Switch back to the actual toolchains
// IsOpenMPOffloading =
- // (IsCuda || IsHIP || IsSYCL || IsOpenMPOffloading) && UseLLVMOffload;
+ // (IsCuda || IsHIP || IsSYCL || IsOpenMPOffloading) &&
+ // UsesLLVMOffloading;
// We currently don't support any kind of mixed offloading.
if (IsOpenMPOffloading)
@@ -1131,7 +1140,6 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
auto &TC = getOffloadToolChain(C.getInputArgs(), Kind, Target,
C.getDefaultToolChain().getTriple());
- llvm::errs() << "Determined TC: " << TC.getArchName() << '\n';
// Emit a warning if the detected CUDA version is too new.
if (Kind == Action::OFK_Cuda && Target.getOS() == llvm::Triple::CUDA) {
@@ -5057,8 +5065,6 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
Action::OFK_OpenMP, Action::OFK_Cuda, Action::OFK_HIP, Action::OFK_SYCL};
for (Action::OffloadKind Kind : OffloadKinds) {
- llvm::errs() << "Offload Kind: " << Action::GetOffloadKindName(Kind) \
- << '\n';
SmallVector<const ToolChain *, 2> ToolChains;
ActionList DeviceActions;
@@ -5071,8 +5077,6 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
types::ID InputType = Input.first;
const Arg *InputArg = Input.second;
- llvm::errs() << "Input Type: " << types::getTypeName(InputType)
- << ", Input Arg: " << InputArg->getAsString(Args) << '\n';
// Allow the toolchain to be active for unsupported file types if we are "offload-cross-compiling" via llvm-offload.
if ((Kind == Action::OFK_Cuda && !types::isCuda(InputType)) ||
@@ -6988,9 +6992,6 @@ std::string Driver::GetClPchPath(Compilation &C, StringRef BaseName) const {
const ToolChain &Driver::getOffloadToolChain(
const llvm::opt::ArgList &Args, const Action::OffloadKind Kind,
const llvm::Triple &Target, const llvm::Triple &AuxTarget) const {
-
- llvm::errs() << "gettting offload TC for Offloading Kind: " << Kind
- << "and target triple: " << Target.str() << '\n';
std::unique_ptr<ToolChain> &TC =
ToolChains[Target.str() + "/" + AuxTarget.str()];
std::unique_ptr<ToolChain> &HostTC = ToolChains[AuxTarget.str()];
@@ -6998,7 +6999,6 @@ const ToolChain &Driver::getOffloadToolChain(
assert(HostTC && "Host toolchain for offloading doesn't exit?");
if (!TC) {
// Detect the toolchain based off of the target operating system.
- llvm::errs() << Target.getOS() << "\n";
switch (Target.getOS()) {
case llvm::Triple::CUDA:
TC = std::make_unique<toolchains::CudaToolChain>(*this, Target, *HostTC,
@@ -7012,7 +7012,7 @@ const ToolChain &Driver::getOffloadToolChain(
TC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(*this, Target,
*HostTC, Args);
else if (Kind == Action::OFK_Cuda)
- //TODO: [h15] figure out if this should be a new TC
+ // TODO: [h15] figure out if this should be a new TC
TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
*HostTC, Args);
break;
@@ -7020,7 +7020,6 @@ const ToolChain &Driver::getOffloadToolChain(
break;
}
}
- llvm::errs() << "Determined Toolchain: " << TC->getArchName() << '\n';
if (!TC) {
// Detect the toolchain based off of the target architecture if that failed.
switch (Target.getArch()) {
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 3a183ea37513a..2c361eacd7646 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -953,11 +953,10 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
// before we -I or -include anything else, because we must pick up the
// CUDA/HIP/SYCL headers from the particular CUDA/ROCm/SYCL installation,
// rather than from e.g. /usr/local/include.
- bool UsesLLVMOffloading = Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false) && getToolChain().getTriple().getEnvironment() == llvm::Triple::LLVM;
- llvm::errs() << "Is Cuda: " << JA.isOffloading(Action::OFK_Cuda) << "\n";
- llvm::errs() << "Offloading? " << JA.getOffloadingDeviceKind() << "\n";
- llvm::errs() << "Triple Env: " << getToolChain().getTriple().getEnvironmentName() << "\n";
+ // bool UsesLLVMOffloading = getToolChain().getTriple().getEnvironment() ==
+ // llvm::Triple::LLVM &&
+ bool UsesLLVMOffloading = Args.hasFlag(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
if (JA.isOffloading(Action::OFK_Cuda) && !UsesLLVMOffloading) {
getToolChain().printVerboseInfo(llvm::errs());
getToolChain().AddCudaIncludeArgs(Args, CmdArgs);
@@ -5187,10 +5186,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
bool IsSYCLDevice = JA.isDeviceOffloading(Action::OFK_SYCL);
bool IsOpenMPDevice = JA.isDeviceOffloading(Action::OFK_OpenMP);
bool IsExtractAPI = isa<ExtractAPIJobAction>(JA);
- bool UsesLLVMOffloading =
- Triple.getEnvironment() == llvm::Triple::LLVM &&
- Args.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
+ // bool UsesLLVMOffloading = Triple.getEnvironment() == llvm::Triple::LLVM &&
+ bool UsesLLVMOffloading = Args.hasFlag(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
bool IsDeviceOffloadAction = !(JA.isDeviceOffloading(Action::OFK_None) ||
JA.isDeviceOffloading(Action::OFK_Host));
bool IsHostOffloadingAction =
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index e2df83b21d76a..3f344dce52fa1 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -301,8 +301,8 @@ CudaInstallationDetector::CudaInstallationDetector(
void CudaInstallationDetector::AddCudaIncludeArgs(
const ArgList &DriverArgs, ArgStringList &CC1Args) const {
- if (HostTriple.getEnvironment() == llvm::Triple::LLVM ||
- DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ // if (HostTriple.getEnvironment() == llvm::Triple::LLVM &&
+ if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
options::OPT_fno_offload_via_llvm, false)) {
if (DriverArgs.hasFlag(options::OPT_offload_inc,
options::OPT_no_offload_inc, true) &&
@@ -318,9 +318,6 @@ void CudaInstallationDetector::AddCudaIncludeArgs(
}
return;
}
- // if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
- // options::OPT_fno_offload_via_llvm, false))
- // return;
if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
// Add cuda_wrappers/* to our system include path. This lets us wrap
@@ -915,14 +912,14 @@ void CudaToolChain::addClangTargetOptions(
BoundArch BA, Action::OffloadKind DeviceOffloadingKind) const {
HostTC.addClangTargetOptions(DriverArgs, CC1Args, BA, DeviceOffloadingKind);
- bool UseLLVMOffload =
- getTriple().getEnvironment() == llvm::Triple::LLVM &&
- DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
- options::OPT_fno_offload_via_llvm, false);
+ // bool UsesLLVMOffloading =
+ // getTriple().getEnvironment() == llvm::Triple::LLVM &&
+ bool UsesLLVMOffloading = DriverArgs.hasFlag(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);
assert((DeviceOffloadingKind == Action::OFK_OpenMP ||
- DeviceOffloadingKind == Action::OFK_Cuda || UseLLVMOffload) &&
+ DeviceOffloadingKind == Action::OFK_Cuda || UsesLLVMOffloading) &&
"Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs.");
CC1Args.append({"-fcuda-is-device", "-mllvm",
@@ -941,7 +938,7 @@ void CudaToolChain::addClangTargetOptions(
DriverArgs.hasArg(options::OPT_S))
return;
- if (UseLLVMOffload)
+ if (UsesLLVMOffloading)
return;
std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch);
@@ -993,13 +990,12 @@ llvm::DenormalMode CudaToolChain::getDefaultDenormalModeForType(
void CudaToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,
ArgStringList &CC1Args) const {
- if (getTriple().getEnvironment() == llvm::Triple::LLVM ||
- DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ // if (getTriple().getEnvironment() == llvm::Triple::LLVM &&
+ if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
options::OPT_fno_offload_via_llvm, false)) {
if (DriverArgs.hasFlag(options::OPT_offload_inc,
options::OPT_no_offload_inc, true) &&
- !DriverArgs.hasArg(options::OPT_nohipwrapperinc) &&
- !DriverArgs.hasArg(options::OPT_nobuiltininc)){
+ !DriverArgs.hasArg(options::OPT_nobuiltininc)) {
CC1Args.append({"-include", "__clang_gpu_device_functions.h"});
SmallString<128> CudaIncludePath(getDriver().ResourceDir);
@@ -1011,9 +1007,6 @@ void CudaToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,
}
return;
}
- // if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
- // options::OPT_fno_offload_via_llvm, false))
- // return;
// Check our CUDA version if we're going to include the CUDA headers.
if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index b1d09ffef919b..10833bbe0eb83 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -235,7 +235,10 @@ HIPAMDToolChain::HIPAMDToolChain(const Driver &D, const llvm::Triple &Triple,
void HIPAMDToolChain::addClangTargetOptions(
const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
BoundArch BA, Action::OffloadKind DeviceOffloadingKind) const {
- bool UsesLLVMOffloading = DriverArgs.hasFlag(options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false) || getTriple().getEnvironment() == llvm::Triple::LLVM;
+ // bool UsesLLVMOffloading = getTriple().getEnvironment() ==
+ // llvm::Triple::LLVM &&
+ bool UsesLLVMOffloading = DriverArgs.hasFlag(
+ options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
assert((DeviceOffloadingKind == Action::OFK_HIP || UsesLLVMOffloading) &&
"Only HIP offloading kinds are supported for GPUs.");
@@ -325,8 +328,8 @@ void HIPAMDToolChain::AddIAMCUIncludeArgs(const ArgList &Args,
void HIPAMDToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
ArgStringList &CC1Args) const {
- if (getTriple().getEnvironment() == llvm::Triple::LLVM &&
- DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
+ // if (getTriple().getEnvironment() == llvm::Triple::LLVM &&
+ if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
options::OPT_fno_offload_via_llvm, false)) {
if (DriverArgs.hasFlag(options::OPT_offload_inc,
options::OPT_no_offload_inc, true) &&
>From e290c3f4389a02f50ca7e36f283ef2a1f1bff926 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Thu, 16 Jul 2026 10:31:19 -0700
Subject: [PATCH 22/23] cleanup
---
.../tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 4 ----
offload/languages/kernel/src/LanguageRegistration.cpp | 10 ----------
offload/languages/kernel/src/LanguageRuntime.cpp | 2 --
3 files changed, 16 deletions(-)
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 3b14168b7b16d..37ceb1b48a6d0 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -799,8 +799,6 @@ wrapDeviceImages(ArrayRef<std::unique_ptr<MemoryBuffer>> Buffers,
M.setTargetTriple(Triple(
Args.getLastArgValue(OPT_host_triple_EQ, sys::getDefaultTargetTriple())));
- llvm::errs() << "Switching on offload kind: " << getOffloadKindName(Kind)
- << "\n";
switch (Kind) {
case OFK_OpenMP:
if (Error Err = offloading::wrapOpenMPBinaries(
@@ -980,8 +978,6 @@ Expected<SmallVector<std::unique_ptr<MemoryBuffer>>>
bundleLinkedOutput(ArrayRef<OffloadingImage> Images, const ArgList &Args,
OffloadKind Kind) {
llvm::TimeTraceScope TimeScope("Bundle linked output");
- llvm::errs() << "Bundling on offload kind: " << getOffloadKindName(Kind)
- << "\n";
if (usesLLVMOffloadWrapper(Images))
return bundleOpenMP(Images);
diff --git a/offload/languages/kernel/src/LanguageRegistration.cpp b/offload/languages/kernel/src/LanguageRegistration.cpp
index 1be65cd63aa34..e108a273f50cd 100644
--- a/offload/languages/kernel/src/LanguageRegistration.cpp
+++ b/offload/languages/kernel/src/LanguageRegistration.cpp
@@ -194,9 +194,6 @@ extern "C" {
void __llvmRegisterFunction(const char *Binary, const char *KernelID,
char *KernelName, const char *KernelName1, int,
uint3 *, uint3 *, dim3 *, dim3 *, int *) {
- // printf("%s :: %p :: %p : %s : %s \n", __PRETTY_FUNCTION__, Binary,
- // KernelID,
- // KernelName, KernelName1);
ol_symbol_handle_t Kernel;
ol_program_handle_t Program = olKGetProgram(Binary);
ol_result_t Result = olGetSymbol(
@@ -207,18 +204,11 @@ void __llvmRegisterFunction(const char *Binary, const char *KernelID,
abort();
}
- // printf("K %p : %p\n", KernelID, Kernel);
olKRegisterKernel(KernelID, Kernel);
}
const char *__llvmRegisterFatBinary(const char *Binary) {
-
const auto *FW = reinterpret_cast<const FatbinWrapperTy *>(Binary);
- // printf("%s : %i : %s (%p:%p) :: %i\n", __PRETTY_FUNCTION__, FW->Magic,
- // FW->Data, FW->Data, FW->DataEnd, FW->Version);
-
- // printf("%s : %s : %lu\n", FW->Data, HIP_FATBIN_MAGIC_STR,
- // HIP_FATBIN_MAGIC_STR_LEN);
if (FW->Magic == 0x466243b1) {
readTUFatbin(Binary, FW);
} else if (FW->Magic == 0x48495046) {
diff --git a/offload/languages/kernel/src/LanguageRuntime.cpp b/offload/languages/kernel/src/LanguageRuntime.cpp
index 43c6e0b994124..4a434da008ec6 100644
--- a/offload/languages/kernel/src/LanguageRuntime.cpp
+++ b/offload/languages/kernel/src/LanguageRuntime.cpp
@@ -174,8 +174,6 @@ Error_t GetDeviceProperties(DeviceProp_t *DeviceProp, int DeviceNo) {
return Success;
}
-// Error_t GetDeviceAttribute()
-
static Error_t getQueueFromStream(Stream_t Stream, ol_queue_handle_t *Queue) {
if (!Stream)
return ErrorInvalidValue;
>From 2cb9f8dd2e36ffe70735da51acd56174d4203721 Mon Sep 17 00:00:00 2001
From: Sophia Herrmann <herrmann15 at llnl.gov>
Date: Sun, 19 Jul 2026 19:19:46 -0700
Subject: [PATCH 23/23] more cleanup
---
clang/lib/Driver/Driver.cpp | 1 -
clang/lib/Driver/ToolChains/Clang.cpp | 1 -
clang/lib/Driver/ToolChains/Cuda.cpp | 1 -
3 files changed, 3 deletions(-)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 2b26147d158a1..a15706cc36df8 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -7012,7 +7012,6 @@ const ToolChain &Driver::getOffloadToolChain(
TC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(*this, Target,
*HostTC, Args);
else if (Kind == Action::OFK_Cuda)
- // TODO: [h15] figure out if this should be a new TC
TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
*HostTC, Args);
break;
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 2c361eacd7646..ee8a52994a56f 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -958,7 +958,6 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
bool UsesLLVMOffloading = Args.hasFlag(
options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
if (JA.isOffloading(Action::OFK_Cuda) && !UsesLLVMOffloading) {
- getToolChain().printVerboseInfo(llvm::errs());
getToolChain().AddCudaIncludeArgs(Args, CmdArgs);
}
if (JA.isOffloading(Action::OFK_HIP) && !UsesLLVMOffloading)
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 3f344dce52fa1..634505861e141 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -416,7 +416,6 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
bool UsesLLVMOffloading = Args.hasFlag(
options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
- assert((UsesLLVMOffloading || TC.getTriple().isNVPTX()) && "Wrong platform");
BoundArch GPUArch;
// If this is a CUDA action we need to extract the device architecture
More information about the cfe-commits
mailing list