[flang-commits] [flang] [flang] Mark loops LICM moves into gang-redundant code of an acc parallel region (PR #231443)
via flang-commits
flang-commits at lists.llvm.org
Tue Oct 13 03:21:44 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-fir-hlfir
Author: Kareem Ergawy (ergawy)
<details>
<summary>Changes</summary>
In a compute region from an acc.parallel construct, code outside a loop with gang-level worksharing is executed redundantly by all gangs. A loop that LICM moved there from inside such a loop was not marked as running in that mode, so later passes could distribute it across gangs. When its result is used in the same kernel, each gang then only computed part of it, giving wrong results.
Mark loops moved into gang-redundant code with acc.gpu_block_redundant so they keep being executed by every gang.
Co-authored by: Claude
---
Full diff: https://github.com/llvm/llvm-project/pull/231443.diff
2 Files Affected:
- (modified) flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp (+26)
- (added) flang/test/Transforms/licm-acc-gang-redundant.fir (+108)
``````````diff
diff --git a/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp b/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp
index f76c8deba5204..8c2413ff1bc80 100644
--- a/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp
@@ -19,7 +19,10 @@
#include "flang/Optimizer/HLFIR/HLFIROps.h"
#include "flang/Optimizer/Support/Utils.h"
#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Dialect/OpenACC/OpenACC.h"
#include "mlir/Dialect/OpenACC/OpenACCUtils.h"
+#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
@@ -248,6 +251,28 @@ static bool canHoistLoad(Operation *op, LoopLikeOpInterface loopLike,
return false;
}
+/// Mark the loops in \p op that it now executes in gang-redundant mode.
+///
+/// In a compute region from an acc.parallel construct, code outside a loop
+/// with gang-level worksharing is executed redundantly by all gangs. A loop
+/// moved into such code, for example out of an scf.parallel, is in
+/// gang-redundant mode: every gang executes all of its iterations. Mark it with
+/// acc.gpu_block_redundant so that it is not later distributed across gangs.
+static void markGangRedundantLoops(Operation *op) {
+ auto computeRegion = op->getParentOfType<acc::ComputeRegionOp>();
+ if (!computeRegion ||
+ computeRegion.getOrigin() != acc::ParallelOp::getOperationName())
+ return;
+
+ op->walk([&](scf::ForOp forOp) {
+ auto parallelOp = forOp->getParentOfType<scf::ParallelOp>();
+ if (parallelOp && computeRegion->isProperAncestor(parallelOp))
+ return;
+ LDBG() << "Marking gang-redundant loop: " << forOp;
+ acc::setGPUBlockRedundantAttr(forOp);
+ });
+}
+
/// Returns true iff hoisting \p op out of a nested region is expected to be
/// inexpensive. This is a cost heuristic only; the safety of the hoisting is
/// established separately.
@@ -454,6 +479,7 @@ void LoopInvariantCodeMotion::runOnOperation() {
};
auto moveOutOfRegion = [&](Operation *op, Region *) {
loopLike.moveOutOfLoop(op);
+ markGangRedundantLoops(op);
};
moveLoopInvariantCode(
diff --git a/flang/test/Transforms/licm-acc-gang-redundant.fir b/flang/test/Transforms/licm-acc-gang-redundant.fir
new file mode 100644
index 0000000000000..5aa77e547550f
--- /dev/null
+++ b/flang/test/Transforms/licm-acc-gang-redundant.fir
@@ -0,0 +1,108 @@
+// RUN: fir-opt -flang-licm --split-input-file %s | FileCheck %s
+
+// In a compute region from acc.parallel, an invariant loop moved out of the
+// scf.parallel runs in gang-redundant mode, so it is marked
+// acc.gpu_block_redundant.
+// CHECK-LABEL: func.func @parallel_origin(
+// CHECK: acc.compute_region
+// CHECK: %[[PROD:.*]] = scf.for {{.*}} -> (f32) {
+// CHECK: } {acc.gpu_block_redundant = #acc.gpu_block_redundant}
+// CHECK: scf.parallel
+// CHECK: arith.mulf %[[PROD]],
+// CHECK: } <{origin = "acc.parallel"}>
+func.func @parallel_origin(%arg0: memref<?xf32>, %arg1: index) {
+ acc.kernel_environment {
+ acc.compute_region ins(%a = %arg0, %n = %arg1) : (memref<?xf32>, index) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c4 = arith.constant 4 : index
+ %one = arith.constant 1.0 : f32
+ %f = arith.constant 0.9 : f32
+ scf.parallel (%i) = (%c0) to (%n) step (%c1) {
+ %prod = scf.for %k = %c0 to %c4 step %c1 iter_args(%acc = %one) -> (f32) {
+ %m = arith.mulf %acc, %f : f32
+ scf.yield %m : f32
+ }
+ %x = memref.load %a[%i] : memref<?xf32>
+ %y = arith.mulf %prod, %x : f32
+ memref.store %y, %a[%i] : memref<?xf32>
+ scf.reduce
+ }
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ }
+ return
+}
+
+// -----
+
+// A compute region from acc.kernels has no gang-redundant code: the loop is
+// moved out but not marked.
+// CHECK-LABEL: func.func @kernels_origin(
+// CHECK: acc.compute_region
+// CHECK: scf.for {{.*}} -> (f32) {
+// CHECK-NOT: acc.gpu_block_redundant
+// CHECK: scf.parallel
+// CHECK: } <{origin = "acc.kernels"}>
+func.func @kernels_origin(%arg0: memref<?xf32>, %arg1: index) {
+ acc.kernel_environment {
+ acc.compute_region ins(%a = %arg0, %n = %arg1) : (memref<?xf32>, index) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c4 = arith.constant 4 : index
+ %one = arith.constant 1.0 : f32
+ %f = arith.constant 0.9 : f32
+ scf.parallel (%i) = (%c0) to (%n) step (%c1) {
+ %prod = scf.for %k = %c0 to %c4 step %c1 iter_args(%acc = %one) -> (f32) {
+ %m = arith.mulf %acc, %f : f32
+ scf.yield %m : f32
+ }
+ %x = memref.load %a[%i] : memref<?xf32>
+ %y = arith.mulf %prod, %x : f32
+ memref.store %y, %a[%i] : memref<?xf32>
+ scf.reduce
+ }
+ acc.yield
+ } <{origin = "acc.kernels"}>
+ }
+ return
+}
+
+// -----
+
+// Moved out of an inner scf.parallel into an outer one, the loop is still
+// within gang-level worksharing, so it is not marked.
+// CHECK-LABEL: func.func @into_outer_parallel(
+// CHECK: acc.compute_region
+// CHECK: scf.parallel
+// CHECK: scf.for {{.*}} -> (f32) {
+// CHECK-NOT: acc.gpu_block_redundant
+// CHECK: scf.parallel
+// CHECK: } <{origin = "acc.parallel"}>
+func.func @into_outer_parallel(%arg0: memref<?x?xf32>, %arg1: index) {
+ acc.kernel_environment {
+ acc.compute_region ins(%a = %arg0, %n = %arg1) : (memref<?x?xf32>, index) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c4 = arith.constant 4 : index
+ %one = arith.constant 1.0 : f32
+ scf.parallel (%j) = (%c0) to (%n) step (%c1) {
+ %jf = arith.index_cast %j : index to i32
+ %fj = arith.sitofp %jf : i32 to f32
+ scf.parallel (%i) = (%c0) to (%n) step (%c1) {
+ %prod = scf.for %k = %c0 to %c4 step %c1 iter_args(%acc = %one) -> (f32) {
+ %m = arith.mulf %acc, %fj : f32
+ scf.yield %m : f32
+ }
+ %x = memref.load %a[%i, %j] : memref<?x?xf32>
+ %y = arith.mulf %prod, %x : f32
+ memref.store %y, %a[%i, %j] : memref<?x?xf32>
+ scf.reduce
+ }
+ scf.reduce
+ }
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ }
+ return
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/231443
More information about the flang-commits
mailing list