[Mlir-commits] [mlir] [mlir][OpenACC] Barrier seq loops that reuse gang-private slots after worker calls (PR #224437)
Delaram Talaashrafi
llvmlistbot at llvm.org
Tue Sep 22 07:47:20 PDT 2026
https://github.com/delaram-talaashrafi updated https://github.com/llvm/llvm-project/pull/224437
>From 525083f6cdbaaa64c7dcc33124ae2f33d0ac916e Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Thu, 17 Sep 2026 14:14:27 -0700
Subject: [PATCH 1/4] [mlir][OpenACC] Barrier seq loops that reuse gang-private
slots after worker calls
A sequential `scf.for` that stores a gang-private value and then calls a
worker- or vector-level acc routine can let threads overlap iterations on
the same workgroup slot. Insert a workgroup barrier at the top of that
loop body so the next iteration's store waits for the previous iteration's
last read.
Skip sequential gang remnants, loops under thread-level worksharing, and
bodies already inside `scf.if`, where a workgroup barrier would deadlock.
---
.../Dialect/OpenACC/Transforms/ACCCGToGPU.cpp | 82 ++++++++
...to-gpu-seq-loop-routine-reuse-barrier.mlir | 188 ++++++++++++++++++
2 files changed, 270 insertions(+)
create mode 100644 mlir/test/Dialect/OpenACC/acc-cg-to-gpu-seq-loop-routine-reuse-barrier.mlir
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
index 90442294d668c..6afde077020be 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
+++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
@@ -397,6 +397,19 @@ static bool isThreadXPrivatize(PrivatizeOp privatize) {
return false;
}
+/// True when \p block is nested inside an `scf.if`, i.e. only part of the
+/// workgroup reaches it, so a workgroup barrier there could deadlock.
+static bool isInsideConditional(Block *block) {
+ if (!block)
+ return false;
+ for (Operation *parent = block->getParentOp();
+ parent && !isa<gpu::LaunchOp, FunctionOpInterface>(parent);
+ parent = parent->getParentOp())
+ if (isa<scf::IfOp>(parent))
+ return true;
+ return false;
+}
+
/// Emits a workgroup-wide GPU barrier.
static void emitGPUBarrierWorkgroup(OpBuilder &builder, Location loc) {
gpu::BarrierOp::create(builder, loc);
@@ -564,6 +577,10 @@ class ACCCGToGPULowering {
/// by a later parallel loop over the same private memory.
PrivateMemScope needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp);
+ /// Whether \p loopOp needs a barrier at the top of its body because a
+ /// gang-private slot it writes is reused by the next iteration.
+ bool needsInLoopReuseBarrier(Operation *loopOp);
+
/// Emit `gpu.all_reduce` for a reduction partial.
void createGPUAllReduceOp(Location loc, Value input, Value memref,
arith::AtomicRMWKind kind,
@@ -2239,6 +2256,63 @@ ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
return storeScope;
}
+bool ACCCGToGPULowering::needsInLoopReuseBarrier(Operation *loopOp) {
+ // A sequential scf.parallel is the remainder of a partitioned gang loop: the
+ // gang iterations this workgroup still runs as a grid stride. Its scalar
+ // stores stay predicated and already reconverge between iterations, so the
+ // reuse hole is the nested scf.for coming from `acc loop seq`.
+ if (scf::ParallelOp par = dyn_cast<scf::ParallelOp>(loopOp))
+ return false;
+
+ // The hazard needs a worker- or vector-level routine call in the body: every
+ // workgroup thread reaches such a call and the callee synchronizes
+ // internally, so threads leave it in no particular order and can then be
+ // running different iterations of this loop at the same time.
+ if (!hasThreadLevelRoutineCall)
+ return false;
+ bool callsThreadLevelRoutine = false;
+ loopOp->walk([&](CallOpInterface callOp) -> WalkResult {
+ if (mlir::acc::GPUParallelDimAttr parDim =
+ getAccRoutineCallParDim(callOp, defaultPolicy)) {
+ if (parDim.isThreadX() || parDim.isThreadY()) {
+ callsThreadLevelRoutine = true;
+ return WalkResult::interrupt();
+ }
+ }
+ return WalkResult::advance();
+ });
+ if (!callsThreadLevelRoutine)
+ return false;
+
+ // Only gang-level privatizations are a single copy shared by the whole
+ // workgroup and reused by every iteration; worker- and thread-private copies
+ // cannot be clobbered by another thread.
+ bool storesGangPrivate = false;
+ loopOp->walk([&](memref::StoreOp storeOp) -> WalkResult {
+ if (getPrivateScopeForMemref(storeOp.getMemref()) != PrivateMemScope::Gang)
+ return WalkResult::advance();
+ storesGangPrivate = true;
+ return WalkResult::interrupt();
+ });
+ if (!storesGangPrivate)
+ return false;
+
+ // A barrier inside the body is only safe when the whole workgroup runs every
+ // iteration, so bail out under any thread-level worksharing loop, whose trip
+ // count varies per thread.
+ for (Operation *p = loopOp->getParentOp();
+ p && p != computeRegion.getOperation(); p = p->getParentOp()) {
+ scf::ParallelOp par = dyn_cast<scf::ParallelOp>(p);
+ if (!par)
+ continue;
+ if (mlir::acc::GPUParallelDimsAttr parDims = mlir::acc::getParDimsAttr(par))
+ for (mlir::acc::GPUParallelDimAttr d : parDims.getArray())
+ if (d.isThreadX() || d.isThreadY())
+ return false;
+ }
+ return true;
+}
+
void ACCCGToGPULowering::processPredicateRegion(
acc::PredicateRegionOp interOp) {
LLVM_DEBUG(llvm::dbgs() << "processing predicate region: ";
@@ -3049,6 +3123,14 @@ void ACCCGToGPULowering::processSeqLoop(LoopOp loopOp) {
assert(blockArgs.size() && "expected block arguments for loop");
mapping.map(blockArgs, newLoop.getBody()->getArguments());
+ // A gang-private slot written here is one copy per workgroup, reused by every
+ // iteration, and a thread that has already started the next iteration would
+ // overwrite the value another thread has not read yet. Separate the
+ // iterations with a barrier at the top of the body.
+ if (needsInLoopReuseBarrier(loopOp.getOperation()) &&
+ !isInsideConditional(rewriter.getInsertionBlock()))
+ emitGPUBarrierWorkgroup(rewriter, loopOp->getLoc());
+
for (auto &bodyOp : loopOp.getBody()->getOperations()) {
if (preProcessedPrivateLocals.contains(&bodyOp))
continue;
diff --git a/mlir/test/Dialect/OpenACC/acc-cg-to-gpu-seq-loop-routine-reuse-barrier.mlir b/mlir/test/Dialect/OpenACC/acc-cg-to-gpu-seq-loop-routine-reuse-barrier.mlir
new file mode 100644
index 0000000000000..0ac7405743ed5
--- /dev/null
+++ b/mlir/test/Dialect/OpenACC/acc-cg-to-gpu-seq-loop-routine-reuse-barrier.mlir
@@ -0,0 +1,188 @@
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(acc-cg-to-gpu))" | FileCheck %s
+
+// A gang-private slot written in a sequential `scf.for` is one copy per
+// workgroup and is reused every iteration. A worker-level acc routine in the
+// body runs on every workgroup thread and synchronizes internally, so threads
+// can leave the call in different iterations. A workgroup barrier at the top
+// of the `scf.for` body orders the next iteration's store after the previous
+// iteration's last read.
+
+module attributes {gpu.container_module} {
+ func.func private @worker_routine(memref<i32>) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_worker]>}
+ acc.routine @acc_routine_worker func(@worker_routine) worker
+
+ // CHECK-LABEL: func.func @seq_for_worker_call_gang_private_reuse
+ // CHECK: gpu.launch
+ // CHECK: scf.for
+ // CHECK-NEXT: gpu.barrier
+ // CHECK-NEXT: memref.store
+ // CHECK-NEXT: func.call @worker_routine
+ func.func @seq_for_worker_call_gang_private_reuse() {
+ %c4 = arith.constant 4 : index
+ %c32 = arith.constant 32 : index
+ %bx = acc.par_width %c4 par_dim(#acc.par_dim<block_x>)
+ %tx = acc.par_width %c32 par_dim(#acc.par_dim<thread_x>)
+ %ty = acc.par_width %c4 par_dim(#acc.par_dim<thread_y>)
+ %priv = acc.privatize : () -> !acc.private_type<memref<i32>>
+ acc.compute_region launch(%kbx = %bx, %ktx = %tx, %kty = %ty)
+ ins(%arg10 = %priv)
+ : (!acc.private_type<memref<i32>>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c0_i32 = arith.constant 0 : i32
+ scf.parallel (%g) = (%c0) to (%kbx) step (%c1) {
+ %pl = acc.private_local %arg10 : (!acc.private_type<memref<i32>>) -> memref<i32>
+ scf.for %i = %c0 to %c2 step %c1 {
+ memref.store %c0_i32, %pl[] : memref<i32>
+ func.call @worker_routine(%pl) : (memref<i32>) -> ()
+ %v = memref.load %pl[] : memref<i32>
+ }
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[block_x]>}
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ return
+ }
+
+ // No worker/vector call: the seq loop stays a single-thread predicated
+ // region in the usual case, and this helper must not insert a body barrier.
+ // CHECK-LABEL: func.func @seq_for_no_routine_no_in_loop_barrier
+ // CHECK: scf.for
+ // CHECK-NEXT: memref.store
+ // CHECK-NOT: gpu.barrier
+ func.func @seq_for_no_routine_no_in_loop_barrier() {
+ %c4 = arith.constant 4 : index
+ %c32 = arith.constant 32 : index
+ %bx = acc.par_width %c4 par_dim(#acc.par_dim<block_x>)
+ %tx = acc.par_width %c32 par_dim(#acc.par_dim<thread_x>)
+ %ty = acc.par_width %c4 par_dim(#acc.par_dim<thread_y>)
+ %priv = acc.privatize : () -> !acc.private_type<memref<i32>>
+ acc.compute_region launch(%kbx = %bx, %ktx = %tx, %kty = %ty)
+ ins(%arg10 = %priv)
+ : (!acc.private_type<memref<i32>>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c0_i32 = arith.constant 0 : i32
+ scf.parallel (%g) = (%c0) to (%kbx) step (%c1) {
+ %pl = acc.private_local %arg10 : (!acc.private_type<memref<i32>>) -> memref<i32>
+ scf.for %i = %c0 to %c2 step %c1 {
+ memref.store %c0_i32, %pl[] : memref<i32>
+ %v = memref.load %pl[] : memref<i32>
+ }
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[block_x]>}
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ return
+ }
+
+ // Sequential scf.parallel is the remainder of a partitioned gang loop, not
+ // `acc loop seq`. Scalar stores already reconverge; do not put a reuse
+ // barrier at the top of that remnant.
+ // CHECK-LABEL: func.func @seq_parallel_remnant_worker_call_no_in_loop_barrier
+ // CHECK: scf.parallel
+ // CHECK-NEXT: memref.store
+ // CHECK-NEXT: func.call @worker_routine
+ func.func @seq_parallel_remnant_worker_call_no_in_loop_barrier() {
+ %c4 = arith.constant 4 : index
+ %c32 = arith.constant 32 : index
+ %bx = acc.par_width %c4 par_dim(#acc.par_dim<block_x>)
+ %tx = acc.par_width %c32 par_dim(#acc.par_dim<thread_x>)
+ %ty = acc.par_width %c4 par_dim(#acc.par_dim<thread_y>)
+ %priv = acc.privatize : () -> !acc.private_type<memref<i32>>
+ acc.compute_region launch(%kbx = %bx, %ktx = %tx, %kty = %ty)
+ ins(%arg10 = %priv)
+ : (!acc.private_type<memref<i32>>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c8 = arith.constant 8 : index
+ %c0_i32 = arith.constant 0 : i32
+ scf.parallel (%g) = (%c0) to (%kbx) step (%c1) {
+ scf.parallel (%rem) = (%g) to (%c8) step (%kbx) {
+ %pl = acc.private_local %arg10 : (!acc.private_type<memref<i32>>) -> memref<i32>
+ memref.store %c0_i32, %pl[] : memref<i32>
+ func.call @worker_routine(%pl) : (memref<i32>) -> ()
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[sequential]>}
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[block_x]>}
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ return
+ }
+
+ // A workgroup barrier inside `scf.if` would hang threads that skip the then
+ // block, so the in-loop reuse barrier is not emitted there.
+ // CHECK-LABEL: func.func @seq_for_worker_call_inside_if_no_in_loop_barrier
+ // CHECK: scf.if
+ // CHECK: scf.for
+ // CHECK-NEXT: memref.store
+ // CHECK-NEXT: func.call @worker_routine
+ func.func @seq_for_worker_call_inside_if_no_in_loop_barrier() {
+ %c4 = arith.constant 4 : index
+ %c32 = arith.constant 32 : index
+ %bx = acc.par_width %c4 par_dim(#acc.par_dim<block_x>)
+ %tx = acc.par_width %c32 par_dim(#acc.par_dim<thread_x>)
+ %ty = acc.par_width %c4 par_dim(#acc.par_dim<thread_y>)
+ %priv = acc.privatize : () -> !acc.private_type<memref<i32>>
+ acc.compute_region launch(%kbx = %bx, %ktx = %tx, %kty = %ty)
+ ins(%arg10 = %priv)
+ : (!acc.private_type<memref<i32>>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c0_i32 = arith.constant 0 : i32
+ scf.parallel (%g) = (%c0) to (%kbx) step (%c1) {
+ %pl = acc.private_local %arg10 : (!acc.private_type<memref<i32>>) -> memref<i32>
+ %cond = arith.cmpi eq, %g, %c0 : index
+ scf.if %cond {
+ scf.for %i = %c0 to %c2 step %c1 {
+ memref.store %c0_i32, %pl[] : memref<i32>
+ func.call @worker_routine(%pl) : (memref<i32>) -> ()
+ }
+ }
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[block_x]>}
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ return
+ }
+
+ // Thread-level worksharing ancestors have per-thread trip counts; a
+ // workgroup barrier in the seq body would deadlock.
+ // CHECK-LABEL: func.func @seq_for_worker_call_under_thread_y_no_in_loop_barrier
+ // CHECK: scf.for
+ // CHECK-NEXT: memref.store
+ // CHECK-NEXT: func.call @worker_routine
+ func.func @seq_for_worker_call_under_thread_y_no_in_loop_barrier() {
+ %c4 = arith.constant 4 : index
+ %c32 = arith.constant 32 : index
+ %bx = acc.par_width %c4 par_dim(#acc.par_dim<block_x>)
+ %tx = acc.par_width %c32 par_dim(#acc.par_dim<thread_x>)
+ %ty = acc.par_width %c4 par_dim(#acc.par_dim<thread_y>)
+ %priv = acc.privatize : () -> !acc.private_type<memref<i32>>
+ acc.compute_region launch(%kbx = %bx, %ktx = %tx, %kty = %ty)
+ ins(%arg10 = %priv)
+ : (!acc.private_type<memref<i32>>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c0_i32 = arith.constant 0 : i32
+ scf.parallel (%g) = (%c0) to (%kbx) step (%c1) {
+ scf.parallel (%w) = (%c0) to (%kty) step (%c1) {
+ %pl = acc.private_local %arg10 : (!acc.private_type<memref<i32>>) -> memref<i32>
+ scf.for %i = %c0 to %c2 step %c1 {
+ memref.store %c0_i32, %pl[] : memref<i32>
+ func.call @worker_routine(%pl) : (memref<i32>) -> ()
+ }
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[thread_y]>}
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[block_x]>}
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ return
+ }
+}
>From f587be9955d97cd6ee6d03a006c266f646e97a6d Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Thu, 17 Sep 2026 14:19:46 -0700
Subject: [PATCH 2/4] Format
---
mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
index 6afde077020be..f449955020e64 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
+++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
@@ -2262,7 +2262,7 @@ bool ACCCGToGPULowering::needsInLoopReuseBarrier(Operation *loopOp) {
// stores stay predicated and already reconverge between iterations, so the
// reuse hole is the nested scf.for coming from `acc loop seq`.
if (scf::ParallelOp par = dyn_cast<scf::ParallelOp>(loopOp))
- return false;
+ return false;
// The hazard needs a worker- or vector-level routine call in the body: every
// workgroup thread reaches such a call and the callee synchronizes
>From 10c39b93b997ee0405ddeb542c541035df48c1ba Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Mon, 21 Sep 2026 13:22:41 -0700
Subject: [PATCH 3/4] Reviews
---
.../Dialect/OpenACC/Transforms/ACCCGToGPU.cpp | 93 +++++++++++++------
1 file changed, 66 insertions(+), 27 deletions(-)
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
index 47d93c1f7765e..f23358ea6179b 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
+++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
@@ -397,15 +397,18 @@ static bool isThreadXPrivatize(PrivatizeOp privatize) {
return false;
}
-/// True when \p block is nested inside an `scf.if`, i.e. only part of the
-/// workgroup reaches it, so a workgroup barrier there could deadlock.
+/// True when \p block is nested in a region that only part of the workgroup
+/// enters, such as an `scf.if` or `fir.if` branch, so a workgroup barrier there
+/// could deadlock. A loop body does not count: every thread agrees on the trip
+/// count here.
static bool isInsideConditional(Block *block) {
if (!block)
return false;
for (Operation *parent = block->getParentOp();
parent && !isa<gpu::LaunchOp, FunctionOpInterface>(parent);
parent = parent->getParentOp())
- if (isa<scf::IfOp>(parent))
+ if (isa<RegionBranchOpInterface>(parent) &&
+ !isa<LoopLikeOpInterface>(parent))
return true;
return false;
}
@@ -583,6 +586,17 @@ class ACCCGToGPULowering {
/// `acc.privatize` that materialized the private buffer for \p memref.
acc::PrivatizeOp getPrivatizeForMemref(Value memref);
+ /// Collect privatize ops of gang- or worker-private stores under \p root.
+ /// Returns Gang if any such store is gang-private, else Worker, else None.
+ PrivateMemScope collectSharedPrivateStores(
+ Operation *root, llvm::SmallPtrSetImpl<Operation *> &storePrivatizes);
+
+ /// True when \p memref is backed by one of \p storePrivatizes at \p
+ /// storeScope.
+ bool isMatchingPrivateMemref(
+ Value memref, PrivateMemScope storeScope,
+ const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes);
+
/// Whether a predicate region needs a barrier before stores that will be read
/// by a later parallel loop over the same private memory.
PrivateMemScope needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp);
@@ -2209,6 +2223,31 @@ acc::PrivatizeOp ACCCGToGPULowering::getPrivatizeForMemref(Value memref) {
return acc::PrivatizeOp();
}
+PrivateMemScope ACCCGToGPULowering::collectSharedPrivateStores(
+ Operation *root, llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) {
+ PrivateMemScope storeScope = PrivateMemScope::None;
+ root->walk([&](memref::StoreOp storeOp) {
+ PrivateMemScope scope = getPrivateScopeForMemref(storeOp.getMemref());
+ if (scope != PrivateMemScope::Gang && scope != PrivateMemScope::Worker)
+ return;
+ if (acc::PrivatizeOp privatize = getPrivatizeForMemref(storeOp.getMemref()))
+ storePrivatizes.insert(privatize.getOperation());
+ // Prefer gang: a workgroup-shared slot is the stronger reuse hazard.
+ if (storeScope == PrivateMemScope::None || scope == PrivateMemScope::Gang)
+ storeScope = scope;
+ });
+ return storeScope;
+}
+
+bool ACCCGToGPULowering::isMatchingPrivateMemref(
+ Value memref, PrivateMemScope storeScope,
+ const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) {
+ if (getPrivateScopeForMemref(memref) != storeScope)
+ return false;
+ acc::PrivatizeOp usePrivatize = getPrivatizeForMemref(memref);
+ return usePrivatize && storePrivatizes.contains(usePrivatize.getOperation());
+}
+
PrivateMemScope
ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
@@ -2220,18 +2259,9 @@ ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
// Check if any op in the predicate region stores to gang- or worker-private
// memory. If not, no barrier is needed.
- PrivateMemScope storeScope = PrivateMemScope::None;
llvm::SmallPtrSet<Operation *, 4> storePrivatizes;
- interOp.getRegion().walk([&](memref::StoreOp storeOp) {
- PrivateMemScope scope = getPrivateScopeForMemref(storeOp.getMemref());
- if (scope != PrivateMemScope::Gang && scope != PrivateMemScope::Worker)
- return WalkResult::advance();
- if (auto privatize = getPrivatizeForMemref(storeOp.getMemref()))
- storePrivatizes.insert(privatize.getOperation());
- if (storeScope == PrivateMemScope::None)
- storeScope = scope;
- return WalkResult::advance();
- });
+ PrivateMemScope storeScope =
+ collectSharedPrivateStores(interOp, storePrivatizes);
if (storeScope == PrivateMemScope::None || storePrivatizes.empty())
return PrivateMemScope::None;
@@ -2250,12 +2280,7 @@ ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
else
return WalkResult::advance();
- PrivateMemScope scope = getPrivateScopeForMemref(memref);
- if (scope != storeScope)
- return WalkResult::advance();
-
- acc::PrivatizeOp usePrivatize = getPrivatizeForMemref(memref);
- if (!usePrivatize || !storePrivatizes.contains(usePrivatize.getOperation()))
+ if (!isMatchingPrivateMemref(memref, storeScope, storePrivatizes))
return WalkResult::advance();
bool insideNestedParallel = false;
@@ -2317,14 +2342,28 @@ bool ACCCGToGPULowering::needsInLoopReuseBarrier(Operation *loopOp) {
// Only gang-level privatizations are a single copy shared by the whole
// workgroup and reused by every iteration; worker- and thread-private copies
// cannot be clobbered by another thread.
- bool storesGangPrivate = false;
- loopOp->walk([&](memref::StoreOp storeOp) -> WalkResult {
- if (getPrivateScopeForMemref(storeOp.getMemref()) != PrivateMemScope::Gang)
- return WalkResult::advance();
- storesGangPrivate = true;
- return WalkResult::interrupt();
+ llvm::SmallPtrSet<Operation *, 4> storedPrivatizes;
+ if (collectSharedPrivateStores(loopOp, storedPrivatizes) !=
+ PrivateMemScope::Gang ||
+ storedPrivatizes.empty())
+ return false;
+
+ // Storing alone is harmless: the next iteration only clobbers a value someone
+ // still needs when the slot is read back in the body, either directly or by a
+ // callee it is handed to.
+ bool reusesGangPrivate = false;
+ loopOp->walk([&](Operation *op) -> WalkResult {
+ if (memref::LoadOp loadOp = dyn_cast<memref::LoadOp>(op))
+ reusesGangPrivate = isMatchingPrivateMemref(
+ loadOp.getMemref(), PrivateMemScope::Gang, storedPrivatizes);
+ else if (CallOpInterface callOp = dyn_cast<CallOpInterface>(op))
+ reusesGangPrivate = llvm::any_of(callOp.getArgOperands(), [&](Value arg) {
+ return isMatchingPrivateMemref(arg, PrivateMemScope::Gang,
+ storedPrivatizes);
+ });
+ return reusesGangPrivate ? WalkResult::interrupt() : WalkResult::advance();
});
- if (!storesGangPrivate)
+ if (!reusesGangPrivate)
return false;
// A barrier inside the body is only safe when the whole workgroup runs every
>From 559965fe7e5170248b48d1ddbdf9449dfe713c36 Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Tue, 22 Sep 2026 07:30:41 -0700
Subject: [PATCH 4/4] Fi
---
mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
index f23358ea6179b..a7ac7b15d4925 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
+++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
@@ -398,9 +398,7 @@ static bool isThreadXPrivatize(PrivatizeOp privatize) {
}
/// True when \p block is nested in a region that only part of the workgroup
-/// enters, such as an `scf.if` or `fir.if` branch, so a workgroup barrier there
-/// could deadlock. A loop body does not count: every thread agrees on the trip
-/// count here.
+/// enters.
static bool isInsideConditional(Block *block) {
if (!block)
return false;
@@ -408,7 +406,7 @@ static bool isInsideConditional(Block *block) {
parent && !isa<gpu::LaunchOp, FunctionOpInterface>(parent);
parent = parent->getParentOp())
if (isa<RegionBranchOpInterface>(parent) &&
- !isa<LoopLikeOpInterface>(parent))
+ !isa<LoopLikeOpInterface, scf::ExecuteRegionOp>(parent))
return true;
return false;
}
More information about the Mlir-commits
mailing list