[flang-commits] [flang] [flang][OpenACC] Allocate bufferized recipe descriptors in device memory (PR #223028)
Slava Zakharin via flang-commits
flang-commits at lists.llvm.org
Fri Sep 11 14:57:46 PDT 2026
https://github.com/vzakhari updated https://github.com/llvm/llvm-project/pull/223028
>From a9edec81dd9983aaa4456e0367a573a6e93e5bf9 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Fri, 11 Sep 2026 13:34:11 -0700
Subject: [PATCH] [flang][OpenACC] Allocate bufferized recipe descriptors in
device memory
ACCRecipeBufferization places the descriptor of a bufferized recipe clause
next to the variable it holds. When the clause sits inside a compute
construct and the variable is already produced by a data entry operation,
that memory is host memory no data clause maps, so device code reading the
descriptor through it gets a host address.
Place the descriptor in the construct region in that case. The region is
device memory and needs no data clause of its own, and the mapped variable
is a legal live-in since a data entry operation produces it. The region
entry block is used rather than the clause operation so that a clause on a
nested loop does not allocate on every iteration.
Assisted-by: Cursor
---
.../include/flang/Optimizer/OpenACC/Passes.td | 8 +-
.../Transforms/ACCRecipeBufferization.cpp | 38 +-
.../Fir/OpenACC/recipe-bufferization.mlir | 327 ++++++++++++++++++
3 files changed, 370 insertions(+), 3 deletions(-)
diff --git a/flang/include/flang/Optimizer/OpenACC/Passes.td b/flang/include/flang/Optimizer/OpenACC/Passes.td
index 0a1f636e1849c..d9bb2ab502e3c 100644
--- a/flang/include/flang/Optimizer/OpenACC/Passes.td
+++ b/flang/include/flang/Optimizer/OpenACC/Passes.td
@@ -56,8 +56,12 @@ def ACCRecipeBufferization
yielded fir.box<T> into it so the region yields a reference to a box.
For acc.private, acc.firstprivate, and acc.reduction operations that use a
- bufferized recipe, the pass allocates a host-side fir.ref<fir.box<T>> before
- the data op and rewires the data op to use the new memory. Other users of
+ bufferized recipe, the pass allocates a fir.ref<fir.box<T>> and rewires the
+ data op to use the new memory. Device code reads the descriptor through
+ that memory, so it is placed on the same side of the host/device boundary
+ as the descriptor value it holds: next to the variable in general, or in
+ the enclosing compute construct region when the variable is already
+ produced by a data entry operation outside the construct. Other users of
the original data operation result (outside the paired compute op) are
updated to load through the reference.
}];
diff --git a/flang/lib/Optimizer/OpenACC/Transforms/ACCRecipeBufferization.cpp b/flang/lib/Optimizer/OpenACC/Transforms/ACCRecipeBufferization.cpp
index 4891bc26e72e1..fa97a097785b9 100644
--- a/flang/lib/Optimizer/OpenACC/Transforms/ACCRecipeBufferization.cpp
+++ b/flang/lib/Optimizer/OpenACC/Transforms/ACCRecipeBufferization.cpp
@@ -93,6 +93,42 @@ static void bufferizeRegionArgsAndYields(mlir::Region ®ion,
}
}
+/// Positions \p builder where the memory holding the bufferized descriptor of
+/// \p clauseOp should be created.
+///
+/// Device code reads the descriptor through this memory, so the memory and the
+/// value stored into it must end up on the same side of the host/device
+/// boundary. Placing it next to the variable - the default - satisfies that for
+/// a host variable, whose memory a data clause can then map.
+///
+/// Device memory is required, and legal, only when all three conditions hold:
+/// - there is an enclosing compute construct, whose region is device memory
+/// needing no data clause of its own. A clause on an orphaned loop or on
+/// the construct itself has none.
+/// - the variable comes from a data entry operation, so the value stored is
+/// a device address rather than a host one.
+/// - that operation is outside the construct, so the value is a legal live-in
+/// and its definition dominates the region entry block.
+///
+/// The entry block is used rather than the clause operation so that a clause on
+/// a nested loop does not allocate on every iteration.
+static void setDescriptorInsertionPoint(mlir::OpBuilder &builder,
+ mlir::Operation *clauseOp,
+ mlir::Value var) {
+ mlir::Operation *varOp = var.getDefiningOp();
+ mlir::Operation *construct =
+ clauseOp->getParentOfType<ACC_COMPUTE_CONSTRUCT_OPS>();
+ if (construct && llvm::isa_and_nonnull<ACC_DATA_ENTRY_OPS>(varOp) &&
+ !construct->isProperAncestor(varOp)) {
+ mlir::Region ®ion = construct->getRegion(0);
+ if (!region.empty()) {
+ builder.setInsertionPointToStart(®ion.front());
+ return;
+ }
+ }
+ builder.setInsertionPointAfterValue(var);
+}
+
template <typename OpTy>
static void updateRecipeUse(mlir::ValueRange operands,
llvm::StringRef recipeSymName,
@@ -106,7 +142,7 @@ static void updateRecipeUse(mlir::ValueRange operands,
mlir::Location loc = op->getLoc();
mlir::OpBuilder builder(op);
- builder.setInsertionPointAfterValue(op.getVar());
+ setDescriptorInsertionPoint(builder, op, op.getVar());
mlir::Value alloca =
BufferizeInterface::placeInMemory(builder, loc, op.getVar());
op.getVarMutable().assign(alloca);
diff --git a/flang/test/Fir/OpenACC/recipe-bufferization.mlir b/flang/test/Fir/OpenACC/recipe-bufferization.mlir
index df3e3306d1195..1ec01b58b4686 100644
--- a/flang/test/Fir/OpenACC/recipe-bufferization.mlir
+++ b/flang/test/Fir/OpenACC/recipe-bufferization.mlir
@@ -314,3 +314,330 @@ func.func @_QPfoo(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
// CHECK: }
// CHECK: return
// CHECK: }
+
+// -----
+
+// A loop reduction on a mapped descriptor. The memory holding the descriptor
+// is created in the construct region, where it is device memory read through
+// the mapped descriptor and needs no data clause of its own.
+
+acc.reduction.recipe @red_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+} combiner {
+^bb0(%lhs: !fir.box<!fir.array<?xi32>>, %rhs: !fir.box<!fir.array<?xi32>>):
+ acc.yield %lhs : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPloop_reduction(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFloop_reductionEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ %1 = acc.copyin var(%0 : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.parallel dataOperands(%1 : !fir.box<!fir.array<?xi32>>) {
+ %2 = acc.reduction var(%1 : !fir.box<!fir.array<?xi32>>) recipe(@red_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop gang reduction(%2 : !fir.box<!fir.array<?xi32>>) control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPloop_reduction(
+// CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+// CHECK: %[[LB:.*]] = arith.constant 1 : i32
+// CHECK: %[[UB:.*]] = arith.constant 32 : i32
+// CHECK: %[[DECL:.*]] = fir.declare %[[ARG0]] {uniq_name = "_QFloop_reductionEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK: %[[MAPPED:.*]] = acc.copyin var(%[[DECL]] : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+// CHECK: acc.parallel dataOperands(%[[MAPPED]] : !fir.box<!fir.array<?xi32>>) {
+// CHECK-NEXT: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: fir.store %[[MAPPED]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK-NEXT: %[[RED:.*]] = acc.reduction varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@red_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.loop gang reduction(%[[RED]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) control(%{{.*}} : i32) = (%[[LB]] : i32) to (%[[UB]] : i32) step (%[[LB]] : i32) {
+// CHECK: } inclusiveUpperbound(array<i1: true>) independent
+// CHECK: acc.yield
+// CHECK: }
+// CHECK: return
+// CHECK: }
+
+// -----
+
+// A reduction carried by a nested loop. The memory belongs at the top of the
+// construct region rather than next to the clause, so that it is not
+// allocated on every iteration of the enclosing loop.
+
+acc.reduction.recipe @red_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+} combiner {
+^bb0(%lhs: !fir.box<!fir.array<?xi32>>, %rhs: !fir.box<!fir.array<?xi32>>):
+ acc.yield %lhs : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPnested_loop_reduction(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFnested_loop_reductionEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ %1 = acc.copyin var(%0 : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.parallel dataOperands(%1 : !fir.box<!fir.array<?xi32>>) {
+ acc.loop gang control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ %2 = acc.reduction var(%1 : !fir.box<!fir.array<?xi32>>) recipe(@red_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop vector reduction(%2 : !fir.box<!fir.array<?xi32>>) control(%arg2 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPnested_loop_reduction(
+// CHECK: %[[MAPPED:.*]] = acc.copyin var(%{{.*}} : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+// CHECK: acc.parallel dataOperands(%[[MAPPED]] : !fir.box<!fir.array<?xi32>>) {
+// CHECK-NEXT: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: fir.store %[[MAPPED]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK-NEXT: acc.loop gang control(
+// CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@red_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.loop vector reduction(%[[RED]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) control(
+// CHECK: return
+// CHECK: }
+
+// -----
+
+// A reduction on the compute construct itself has to keep its memory outside
+// the construct: the clause is an operand of the construct and so cannot refer
+// to a value defined in its region.
+
+acc.reduction.recipe @red_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+} combiner {
+^bb0(%lhs: !fir.box<!fir.array<?xi32>>, %rhs: !fir.box<!fir.array<?xi32>>):
+ acc.yield %lhs : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPconstruct_reduction(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %0 = fir.declare %arg0 {uniq_name = "_QFconstruct_reductionEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ %1 = acc.copyin var(%0 : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+ %2 = acc.reduction var(%1 : !fir.box<!fir.array<?xi32>>) recipe(@red_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.parallel dataOperands(%1 : !fir.box<!fir.array<?xi32>>) reduction(%2 : !fir.box<!fir.array<?xi32>>) {
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPconstruct_reduction(
+// CHECK: %[[MAPPED:.*]] = acc.copyin var(%{{.*}} : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+// CHECK: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK: fir.store %[[MAPPED]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@red_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.parallel dataOperands(%[[MAPPED]] : !fir.box<!fir.array<?xi32>>) reduction(%[[RED]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
+
+// -----
+
+// A descriptor that is not mapped keeps its memory outside the construct: it
+// is not a live-in of the region because only private clauses use it, and
+// storing it in the region would make it one.
+
+acc.private.recipe @priv_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPunmapped_private(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFunmapped_privateEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ acc.parallel {
+ %1 = acc.private var(%0 : !fir.box<!fir.array<?xi32>>) recipe(@priv_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop gang private(%1 : !fir.box<!fir.array<?xi32>>) control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPunmapped_private(
+// CHECK: %[[DECL:.*]] = fir.declare %{{.*}} {uniq_name = "_QFunmapped_privateEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK: fir.store %[[DECL]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.parallel {
+// CHECK-NEXT: %[[PRIV:.*]] = acc.private varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@priv_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.loop gang private(%[[PRIV]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) control(
+
+// -----
+
+// Placement follows the descriptor value, not the kind of clause: a private
+// clause on a mapped descriptor gets its memory in the construct region for
+// the same reason a reduction does.
+
+acc.private.recipe @priv_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPmapped_private(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFmapped_privateEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ %1 = acc.copyin var(%0 : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.parallel dataOperands(%1 : !fir.box<!fir.array<?xi32>>) {
+ %2 = acc.private var(%1 : !fir.box<!fir.array<?xi32>>) recipe(@priv_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop gang private(%2 : !fir.box<!fir.array<?xi32>>) control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPmapped_private(
+// CHECK: %[[MAPPED:.*]] = acc.copyin var(%{{.*}} : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+// CHECK: acc.parallel dataOperands(%[[MAPPED]] : !fir.box<!fir.array<?xi32>>) {
+// CHECK-NEXT: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: fir.store %[[MAPPED]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK-NEXT: %[[PRIV:.*]] = acc.private varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@priv_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.loop gang private(%[[PRIV]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) control(
+
+// -----
+
+// The same for firstprivate.
+
+acc.firstprivate.recipe @fp_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+} copy {
+^bb0(%src: !fir.box<!fir.array<?xi32>>, %dst: !fir.box<!fir.array<?xi32>>):
+ acc.terminator
+}
+func.func @_QPmapped_firstprivate(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFmapped_firstprivateEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ %1 = acc.copyin var(%0 : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.parallel dataOperands(%1 : !fir.box<!fir.array<?xi32>>) {
+ %2 = acc.firstprivate var(%1 : !fir.box<!fir.array<?xi32>>) recipe(@fp_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop gang firstprivate(%2 : !fir.box<!fir.array<?xi32>>) control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPmapped_firstprivate(
+// CHECK: %[[MAPPED:.*]] = acc.copyin var(%{{.*}} : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+// CHECK: acc.parallel dataOperands(%[[MAPPED]] : !fir.box<!fir.array<?xi32>>) {
+// CHECK-NEXT: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: fir.store %[[MAPPED]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK-NEXT: %[[FP:.*]] = acc.firstprivate varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@fp_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.loop gang firstprivate(%[[FP]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) control(
+
+// -----
+
+// A clause that refers to a mapped descriptor through a declare in the region
+// keeps the default placement next to that declare. The value stored is the
+// declare result, so the memory cannot be hoisted above it, and the declare
+// is already inside the region.
+
+acc.private.recipe @priv_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPmapped_private_via_declare(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFmapped_private_via_declareEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ %1 = acc.copyin var(%0 : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.parallel dataOperands(%1 : !fir.box<!fir.array<?xi32>>) {
+ %2 = fir.declare %1 {uniq_name = "_QFmapped_private_via_declareEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ %3 = acc.private var(%2 : !fir.box<!fir.array<?xi32>>) recipe(@priv_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop gang private(%3 : !fir.box<!fir.array<?xi32>>) control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPmapped_private_via_declare(
+// CHECK: %[[MAPPED:.*]] = acc.copyin var(%{{.*}} : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+// CHECK: acc.parallel dataOperands(%[[MAPPED]] : !fir.box<!fir.array<?xi32>>) {
+// CHECK-NEXT: %[[INNER:.*]] = fir.declare %[[MAPPED]] {{.*}} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: fir.store %[[INNER]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK-NEXT: %[[PRIV:.*]] = acc.private varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@priv_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+
+// -----
+
+// A data entry operation inside the construct also keeps the default
+// placement: hoisting the memory to the top of the region would place it
+// above the value it stores.
+
+acc.reduction.recipe @red_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+} combiner {
+^bb0(%lhs: !fir.box<!fir.array<?xi32>>, %rhs: !fir.box<!fir.array<?xi32>>):
+ acc.yield %lhs : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPin_region_data_entry(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFin_region_data_entryEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ acc.parallel {
+ %1 = acc.copyin var(%0 : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+ %2 = acc.reduction var(%1 : !fir.box<!fir.array<?xi32>>) recipe(@red_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop gang reduction(%2 : !fir.box<!fir.array<?xi32>>) control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPin_region_data_entry(
+// CHECK: acc.parallel {
+// CHECK-NEXT: %[[MAPPED:.*]] = acc.copyin var(%{{.*}} : !fir.box<!fir.array<?xi32>>) dataClause(acc_copy) implicit(true) name("r") -> !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT: fir.store %[[MAPPED]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK-NEXT: %[[RED:.*]] = acc.reduction varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@red_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+
+// -----
+
+// A reduction on a descriptor that is not mapped keeps the default placement
+// in host code, like an unmapped private clause.
+//
+// This shape does not arise from a compiler pipeline that applies implicit
+// data clauses, because any aggregate live-in receives one, so a reduction on
+// a box is already mapped when this pass runs. It would not work if it did
+// arise: a reduction gets no initial value mapping, so nothing maps the
+// memory wherever it is placed. The case pins the condition for the reduction
+// clause rather than describing a working end state.
+
+acc.reduction.recipe @red_box_Uxi32 : !fir.box<!fir.array<?xi32>> reduction_operator <add> init {
+^bb0(%arg0: !fir.box<!fir.array<?xi32>>):
+ acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
+} combiner {
+^bb0(%lhs: !fir.box<!fir.array<?xi32>>, %rhs: !fir.box<!fir.array<?xi32>>):
+ acc.yield %lhs : !fir.box<!fir.array<?xi32>>
+}
+func.func @_QPunmapped_reduction(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+ %c1_i32 = arith.constant 1 : i32
+ %c32_i32 = arith.constant 32 : i32
+ %0 = fir.declare %arg0 {uniq_name = "_QFunmapped_reductionEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+ acc.parallel {
+ %1 = acc.reduction var(%0 : !fir.box<!fir.array<?xi32>>) recipe(@red_box_Uxi32) name("r") -> !fir.box<!fir.array<?xi32>>
+ acc.loop gang reduction(%1 : !fir.box<!fir.array<?xi32>>) control(%arg1 : i32) = (%c1_i32 : i32) to (%c32_i32 : i32) step (%c1_i32 : i32) {
+ acc.yield
+ } inclusiveUpperbound(array<i1: true>) independent
+ acc.yield
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QPunmapped_reduction(
+// CHECK: %[[DECL:.*]] = fir.declare %{{.*}} {uniq_name = "_QFunmapped_reductionEr"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK: %[[SLOT:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK: fir.store %[[DECL]] to %[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.parallel {
+// CHECK-NEXT: %[[RED:.*]] = acc.reduction varPtr(%[[SLOT]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) recipe(@red_box_Uxi32) name("r") -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: acc.loop gang reduction(%[[RED]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) control(
More information about the flang-commits
mailing list