[flang-commits] [flang] [flang] Add a pass to get OpenACC device ptr for CUDA kernel (PR #212299)
via flang-commits
flang-commits at lists.llvm.org
Mon Jul 27 10:01:44 PDT 2026
https://github.com/yebinchon created https://github.com/llvm/llvm-project/pull/212299
When a CUDA kernel is launched inside an OpenACC data region, it does not properly get the device pointers and instead uses host data. This PR adds a pass to get the device pointers set up by the OpenACC data construct and pass them explicitly to the CUDA kernel.
>From 768cd898457bc1b7f9829517b38845de5ea7d775 Mon Sep 17 00:00:00 2001
From: Yebin Chon <ychon at nvidia.com>
Date: Mon, 27 Jul 2026 09:50:52 -0700
Subject: [PATCH] [flang] Add a pass to get device pointer set up by OpenACC
for CUDA kernel
---
.../include/flang/Optimizer/OpenACC/Passes.td | 28 +++
.../Transforms/ACCDevicePtrToCUFKernel.cpp | 206 ++++++++++++++++++
.../OpenACC/Transforms/CMakeLists.txt | 2 +
3 files changed, 236 insertions(+)
create mode 100644 flang/lib/Optimizer/OpenACC/Transforms/ACCDevicePtrToCUFKernel.cpp
diff --git a/flang/include/flang/Optimizer/OpenACC/Passes.td b/flang/include/flang/Optimizer/OpenACC/Passes.td
index 0c726b7e5cb86..ea220c818eadd 100644
--- a/flang/include/flang/Optimizer/OpenACC/Passes.td
+++ b/flang/include/flang/Optimizer/OpenACC/Passes.td
@@ -95,4 +95,32 @@ def ACCOptimizeFirstprivateMap
let dependentDialects = ["mlir::acc::OpenACCDialect", "fir::FIROpsDialect"];
}
+def ACCDevicePtrToCUFKernel
+ : Pass<"acc-device-ptr-to-cuf-kernel", "mlir::ModuleOp"> {
+ let summary = "Pass device addresses to CUDA Fortran kernels launched inside "
+ "OpenACC data regions";
+ let description = [{
+ A CUDA Fortran kernel launched inside an OpenACC data region must receive
+ the device address of any host variable that OpenACC has made present, not
+ the host address. Otherwise the kernel dereferences a host pointer, which is
+ only valid on shared-memory/unified-addressing systems and is illegal on a
+ device with a separate address space.
+
+ For each cuf.kernel_launch whose reference arguments resolve to a variable
+ mapped by an enclosing acc.data region, this pass wraps the launch in an
+ acc.host_data construct with acc.use_device operands for the mapped
+ variables, and rebuilds the launch argument addressing on top of the
+ use_device result so the kernel receives the device pointer.
+
+ Both directly-addressed variables (e.g. static arrays, whose data address is
+ the mapped varPtr) and descriptor-based variables (e.g. allocatables and
+ pointers, whose data address is box_addr(load(<descriptor>))) are handled.
+
+ This must run before the pass that lowers cuf.kernel_launch to
+ gpu.launch_func.
+ }];
+ let dependentDialects = ["mlir::acc::OpenACCDialect", "fir::FIROpsDialect",
+ "cuf::CUFDialect"];
+}
+
#endif // FORTRAN_OPTIMIZER_OPENACC_PASSES
diff --git a/flang/lib/Optimizer/OpenACC/Transforms/ACCDevicePtrToCUFKernel.cpp b/flang/lib/Optimizer/OpenACC/Transforms/ACCDevicePtrToCUFKernel.cpp
new file mode 100644
index 0000000000000..863a2490412a9
--- /dev/null
+++ b/flang/lib/Optimizer/OpenACC/Transforms/ACCDevicePtrToCUFKernel.cpp
@@ -0,0 +1,206 @@
+//===- ACCDevicePtrToCUFKernel.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A CUDA Fortran kernel launched inside an OpenACC data
+// region must receive the device address of any host variable that OpenACC has
+// made present, not the host address. This pass wraps each
+// cuf.kernel_launch in an acc.host_data construct with acc.use_device operands
+// for the mapped host variables, and rebuilds the launch's argument addressing
+// on top of the use_device result. The host_data/use_device lowering then
+// materializes the present-table device pointer, and any array-section
+// addressing is recomputed on the device pointer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Optimizer/OpenACC/Passes.h"
+#include "mlir/Dialect/OpenACC/OpenACC.h"
+#include "mlir/IR/IRMapping.h"
+#include "mlir/Pass/Pass.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace fir {
+namespace acc {
+#define GEN_PASS_DEF_ACCDEVICEPTRTOCUFKERNEL
+#include "flang/Optimizer/OpenACC/Passes.h.inc"
+} // namespace acc
+} // namespace fir
+
+using namespace mlir;
+
+namespace {
+
+/// Walk down an addressing chain to the underlying variable that OpenACC maps
+/// as a data-clause pointer
+static Value getMappedVar(Value value) {
+ while (true) {
+ if (auto convert = value.getDefiningOp<fir::ConvertOp>()) {
+ value = convert.getValue();
+ continue;
+ }
+ if (auto coor = value.getDefiningOp<fir::ArrayCoorOp>()) {
+ value = coor.getMemref();
+ continue;
+ }
+ if (auto coor = value.getDefiningOp<fir::CoordinateOp>()) {
+ value = coor.getRef();
+ continue;
+ }
+ if (auto designate = value.getDefiningOp<hlfir::DesignateOp>()) {
+ value = designate.getMemref();
+ continue;
+ }
+ // Descriptor-based (allocatable/pointer) variables: the data address is
+ // extracted from the descriptor via box_addr(load(<descriptor ref>)). Peel
+ // both so the walk reaches the descriptor variable, which is what OpenACC
+ // maps as the data-clause varPtr for such variables.
+ if (auto boxAddr = value.getDefiningOp<fir::BoxAddrOp>()) {
+ value = boxAddr.getVal();
+ continue;
+ }
+ if (auto load = value.getDefiningOp<fir::LoadOp>()) {
+ // Only a load that produces a descriptor is part of the addressing
+ // chain; scalar loads are ordinary values, not addressing steps.
+ if (mlir::isa<fir::BaseBoxType>(load.getType())) {
+ value = load.getMemref();
+ continue;
+ }
+ }
+ if (isa_and_nonnull<fir::DeclareOp, hlfir::DeclareOp>(
+ value.getDefiningOp()))
+ return value;
+ return {};
+ }
+}
+
+/// Checks if mappedVar is present due to an enclosing acc.data region.
+static bool isMappedInEnclosingAccData(Value mappedVar,
+ cuf::KernelLaunchOp launch) {
+ if(!mappedVar)
+ return false;
+ for (auto dataOp = launch->getParentOfType<acc::DataOp>(); dataOp;
+ dataOp = dataOp->getParentOfType<acc::DataOp>()) {
+ for (Value dataOperand : dataOp.getDataClauseOperands()) {
+ if (Value hostVar = acc::getVar(dataOperand.getDefiningOp()))
+ if (getMappedVar(hostVar) == mappedVar)
+ return true;
+ }
+ }
+ return false;
+}
+
+/// Reconstructs the addressing chain that produced `value` from `mappedVar`,
+/// substituting `deviceVar` for `mappedVar`. Only addressing ops are cloned;
+/// everything else (constants, shapes, ...) is reused as a live-in. New ops are
+/// created at `builder`'s current insertion point.
+static Value rebuildOnDevice(OpBuilder &builder, Value value, Value mappedVar,
+ Value deviceVar) {
+ if (value == mappedVar)
+ return deviceVar;
+
+ Operation *def = value.getDefiningOp();
+ if (!def || !isa<fir::ConvertOp, fir::ArrayCoorOp, fir::CoordinateOp,
+ hlfir::DesignateOp, fir::BoxAddrOp, fir::LoadOp>(def))
+ return value;
+
+ // Mirror getMappedVar: only a descriptor load is part of the addressing
+ // chain and must be rebuilt on the device descriptor; any other load is a
+ // live-in and is reused as-is.
+ if (auto load = dyn_cast<fir::LoadOp>(def))
+ if (!mlir::isa<fir::BaseBoxType>(load.getType()))
+ return value;
+
+ IRMapping map;
+ for (Value operand : def->getOperands())
+ map.map(operand, rebuildOnDevice(builder, operand, mappedVar, deviceVar));
+ return builder.clone(*def, map)->getResult(0);
+}
+
+class ACCDevicePtrToCUFKernel
+ : public fir::acc::impl::ACCDevicePtrToCUFKernelBase<
+ ACCDevicePtrToCUFKernel> {
+public:
+ using fir::acc::impl::ACCDevicePtrToCUFKernelBase<
+ ACCDevicePtrToCUFKernel>::ACCDevicePtrToCUFKernelBase;
+
+ void runOnOperation() override {
+ llvm::SmallVector<cuf::KernelLaunchOp> launches;
+ getOperation().walk(
+ [&](cuf::KernelLaunchOp launch) { launches.push_back(launch); });
+
+ for (cuf::KernelLaunchOp launch : launches)
+ rewriteLaunch(launch);
+ }
+
+private:
+ void rewriteLaunch(cuf::KernelLaunchOp launch) {
+ // Collect kernel arguments that are references to a host variable made
+ // present by an enclosing acc.data region.
+ struct MappedArg {
+ OpOperand *operand;
+ Value mappedVar;
+ };
+ llvm::SmallVector<MappedArg> mappedArgs;
+ llvm::SetVector<Value> mappedVars;
+
+ for (OpOperand &operand : launch.getArgsMutable()) {
+ Value arg = operand.get();
+ if (!fir::isa_ref_type(arg.getType()))
+ continue;
+ Value mappedVar = getMappedVar(arg);
+ if (!isMappedInEnclosingAccData(mappedVar, launch))
+ continue;
+ mappedArgs.push_back({&operand, mappedVar});
+ mappedVars.insert(mappedVar);
+ }
+
+ if (mappedArgs.empty())
+ return;
+
+ OpBuilder builder(launch);
+ Location loc = launch.getLoc();
+
+ // One acc.use_device per distinct mapped variable, emitted before the
+ // launch so it dominates the host_data region.
+ llvm::DenseMap<Value, Value> deviceVars;
+ llvm::SmallVector<Value> dataOperands;
+ for (Value mappedVar : mappedVars) {
+ Value deviceVar = acc::UseDeviceOp::create(builder, loc, mappedVar,
+ /*structured=*/true,
+ /*implicit=*/false)
+ .getAccVar();
+ deviceVars[mappedVar] = deviceVar;
+ dataOperands.push_back(deviceVar);
+ }
+
+ // Wrap the launch in an acc.host_data region.
+ auto hostData =
+ acc::HostDataOp::create(builder, loc, /*ifCond=*/Value{}, dataOperands);
+ Block *body = builder.createBlock(&hostData.getRegion());
+ builder.setInsertionPointToStart(body);
+ Operation *terminator = acc::TerminatorOp::create(builder, loc);
+ launch->moveBefore(terminator);
+
+ // Recompute each mapped argument's address on the device pointer.
+ builder.setInsertionPoint(launch);
+ for (MappedArg &mappedArg : mappedArgs) {
+ Value arg = mappedArg.operand->get();
+ Value deviceVar = deviceVars[mappedArg.mappedVar];
+ mappedArg.operand->assign(
+ rebuildOnDevice(builder, arg, mappedArg.mappedVar, deviceVar));
+ }
+ }
+};
+
+} // namespace
diff --git a/flang/lib/Optimizer/OpenACC/Transforms/CMakeLists.txt b/flang/lib/Optimizer/OpenACC/Transforms/CMakeLists.txt
index 5bf4e629861cf..7dd2468dd27c6 100644
--- a/flang/lib/Optimizer/OpenACC/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenACC/Transforms/CMakeLists.txt
@@ -1,5 +1,6 @@
add_flang_library(FIROpenACCTransforms
ACCDeclareActionConversion.cpp
+ ACCDevicePtrToCUFKernel.cpp
ACCInitializeFIRAnalyses.cpp
ACCOptimizeFirstprivateMap.cpp
ACCRecipeBufferization.cpp
@@ -9,6 +10,7 @@ add_flang_library(FIROpenACCTransforms
FIROpenACCPassesIncGen
LINK_LIBS
+ CUFDialect
FIRAnalysis
FIRBuilder
FIRDialect
More information about the flang-commits
mailing list