[Mlir-commits] [mlir] [mlir][OpenACC] Match reuse barriers to the reused private scope (PR #225570)
Delaram Talaashrafi
llvmlistbot at llvm.org
Thu Sep 24 07:38:17 PDT 2026
https://github.com/delaram-talaashrafi updated https://github.com/llvm/llvm-project/pull/225570
>From 0b4f66a7f6e2c5f5c761867dee4489ddf57ed7bf Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Tue, 22 Sep 2026 18:28:10 -0700
Subject: [PATCH] [mlir][OpenACC] Match reuse barriers to the reused private
scope
A region that writes both gang- and worker-private slots was considered
to be a gang scope and a barrier was missed. This change keeps the two
store sets separate and pick the barrier from the scope that is actually reused.
---
.../Dialect/OpenACC/Transforms/ACCCGToGPU.cpp | 97 ++++++++++++-------
...to-gpu-predicate-region-reuse-barrier.mlir | 53 ++++++++++
2 files changed, 113 insertions(+), 37 deletions(-)
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
index a7ac7b15d4925..b8681f775880e 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
+++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCCGToGPU.cpp
@@ -584,10 +584,13 @@ class ACCCGToGPULowering {
/// `acc.privatize` that materialized the private buffer for \p memref.
acc::PrivatizeOp getPrivatizeForMemref(Value memref);
- /// Collect privatize ops of gang- or worker-private stores under \p root.
- /// Returns Gang if any such store is gang-private, else Worker, else None.
- PrivateMemScope collectSharedPrivateStores(
- Operation *root, llvm::SmallPtrSetImpl<Operation *> &storePrivatizes);
+ /// Collect privatize ops of the shared private buffers stored to under \p
+ /// root, keeping the gang- and worker-private ones apart: the two scopes need
+ /// different barriers, so a store at one scope must not stand in for the
+ /// other.
+ void collectSharedPrivateStores(
+ Operation *root, llvm::SmallPtrSetImpl<Operation *> &gangPrivatizes,
+ llvm::SmallPtrSetImpl<Operation *> &workerPrivatizes);
/// True when \p memref is backed by one of \p storePrivatizes at \p
/// storeScope.
@@ -595,6 +598,13 @@ class ACCCGToGPULowering {
Value memref, PrivateMemScope storeScope,
const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes);
+ /// True when a parallel region under \p seqLoopOp, outside \p skipOp, uses a
+ /// private buffer at \p storeScope backed by one of \p storePrivatizes.
+ bool hasParallelPrivateReuse(
+ LoopLikeOpInterface seqLoopOp, Operation *skipOp,
+ PrivateMemScope storeScope,
+ const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes);
+
/// Whether a predicate region needs a barrier before stores that will be read
/// by a later parallel loop over the same private memory.
PrivateMemScope needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp);
@@ -2221,20 +2231,21 @@ acc::PrivatizeOp ACCCGToGPULowering::getPrivatizeForMemref(Value memref) {
return acc::PrivatizeOp();
}
-PrivateMemScope ACCCGToGPULowering::collectSharedPrivateStores(
- Operation *root, llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) {
- PrivateMemScope storeScope = PrivateMemScope::None;
+void ACCCGToGPULowering::collectSharedPrivateStores(
+ Operation *root, llvm::SmallPtrSetImpl<Operation *> &gangPrivatizes,
+ llvm::SmallPtrSetImpl<Operation *> &workerPrivatizes) {
root->walk([&](memref::StoreOp storeOp) {
PrivateMemScope scope = getPrivateScopeForMemref(storeOp.getMemref());
if (scope != PrivateMemScope::Gang && scope != PrivateMemScope::Worker)
return;
- if (acc::PrivatizeOp privatize = getPrivatizeForMemref(storeOp.getMemref()))
- storePrivatizes.insert(privatize.getOperation());
- // Prefer gang: a workgroup-shared slot is the stronger reuse hazard.
- if (storeScope == PrivateMemScope::None || scope == PrivateMemScope::Gang)
- storeScope = scope;
+ acc::PrivatizeOp privatize = getPrivatizeForMemref(storeOp.getMemref());
+ if (!privatize)
+ return;
+ if (scope == PrivateMemScope::Gang)
+ gangPrivatizes.insert(privatize.getOperation());
+ else
+ workerPrivatizes.insert(privatize.getOperation());
});
- return storeScope;
}
bool ACCCGToGPULowering::isMatchingPrivateMemref(
@@ -2246,28 +2257,17 @@ bool ACCCGToGPULowering::isMatchingPrivateMemref(
return usePrivatize && storePrivatizes.contains(usePrivatize.getOperation());
}
-PrivateMemScope
-ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
-
- // Check if we need a pre-predicate barrier first.
- // Next, check if the barrier should be gang- or worker-level.
- LoopLikeOpInterface seqLoopOp = findFirstSequentialLoop(interOp);
- if (!seqLoopOp)
- return PrivateMemScope::None;
-
- // Check if any op in the predicate region stores to gang- or worker-private
- // memory. If not, no barrier is needed.
- llvm::SmallPtrSet<Operation *, 4> storePrivatizes;
- PrivateMemScope storeScope =
- collectSharedPrivateStores(interOp, storePrivatizes);
- if (storeScope == PrivateMemScope::None || storePrivatizes.empty())
- return PrivateMemScope::None;
+bool ACCCGToGPULowering::hasParallelPrivateReuse(
+ LoopLikeOpInterface seqLoopOp, Operation *skipOp,
+ PrivateMemScope storeScope,
+ const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) {
+ if (storePrivatizes.empty())
+ return false;
- // Check that there is a subsequent parallel region that uses private memory
bool hasParallelPrivateUse = false;
seqLoopOp.getOperation()->walk([&](Operation *op) {
// Ignore loads inside the predicate.
- if (interOp->isAncestor(op))
+ if (skipOp->isAncestor(op))
return WalkResult::advance();
Value memref;
@@ -2303,10 +2303,34 @@ ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
return WalkResult::interrupt();
});
- if (!hasParallelPrivateUse)
+ return hasParallelPrivateUse;
+}
+
+PrivateMemScope
+ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
+
+ // Check if we need a pre-predicate barrier first.
+ // Next, check if the barrier should be gang- or worker-level.
+ LoopLikeOpInterface seqLoopOp = findFirstSequentialLoop(interOp);
+ if (!seqLoopOp)
return PrivateMemScope::None;
- return storeScope;
+ // Check if any op in the predicate region stores to gang- or worker-private
+ // memory. If not, no barrier is needed.
+ llvm::SmallPtrSet<Operation *, 4> gangPrivatizes, workerPrivatizes;
+ collectSharedPrivateStores(interOp, gangPrivatizes, workerPrivatizes);
+
+ // A reused gang slot needs the workgroup barrier, which also reconverges the
+ // worker rows, so look for that hazard first. A gang store whose slot is
+ // never read back does not stand in for a worker one: fall through and check
+ // the worker slots on their own, otherwise their per-row barrier is lost.
+ if (hasParallelPrivateReuse(seqLoopOp, interOp, PrivateMemScope::Gang,
+ gangPrivatizes))
+ return PrivateMemScope::Gang;
+ if (hasParallelPrivateReuse(seqLoopOp, interOp, PrivateMemScope::Worker,
+ workerPrivatizes))
+ return PrivateMemScope::Worker;
+ return PrivateMemScope::None;
}
bool ACCCGToGPULowering::needsInLoopReuseBarrier(Operation *loopOp) {
@@ -2340,10 +2364,9 @@ bool ACCCGToGPULowering::needsInLoopReuseBarrier(Operation *loopOp) {
// Only gang-level privatizations are a single copy shared by the whole
// workgroup and reused by every iteration; worker- and thread-private copies
// cannot be clobbered by another thread.
- llvm::SmallPtrSet<Operation *, 4> storedPrivatizes;
- if (collectSharedPrivateStores(loopOp, storedPrivatizes) !=
- PrivateMemScope::Gang ||
- storedPrivatizes.empty())
+ llvm::SmallPtrSet<Operation *, 4> storedPrivatizes, workerPrivatizes;
+ collectSharedPrivateStores(loopOp, storedPrivatizes, workerPrivatizes);
+ if (storedPrivatizes.empty())
return false;
// Storing alone is harmless: the next iteration only clobbers a value someone
diff --git a/mlir/test/Dialect/OpenACC/acc-cg-to-gpu-predicate-region-reuse-barrier.mlir b/mlir/test/Dialect/OpenACC/acc-cg-to-gpu-predicate-region-reuse-barrier.mlir
index 57459666c326a..09075bf3a469d 100644
--- a/mlir/test/Dialect/OpenACC/acc-cg-to-gpu-predicate-region-reuse-barrier.mlir
+++ b/mlir/test/Dialect/OpenACC/acc-cg-to-gpu-predicate-region-reuse-barrier.mlir
@@ -81,3 +81,56 @@ func.func @no_reuse_barrier_outside_seq_loop() {
}
return
}
+
+// Predicate stores to both a gang-private slot (never reused in parallel) and a
+// worker-private slot (reused by a nested vector loop). The gang store must
+// not hide the worker WAR: emit a per-row pre-store barrier, not a workgroup
+// one and not no barrier at all.
+// CHECK-LABEL: func.func @mixed_gang_store_worker_reuse
+// CHECK: gpu.launch
+// CHECK: scf.parallel
+// CHECK: gpu.barrier scope <subgroup>
+// CHECK-NEXT: scf.if
+// CHECK: gpu.barrier scope <subgroup>
+func.func @mixed_gang_store_worker_reuse() {
+ %c256_pw = arith.constant 256 : index
+ %c4_pw = arith.constant 4 : index
+ %c32_pw = arith.constant 32 : index
+ %par_bx = acc.par_width %c256_pw par_dim(#acc.par_dim<block_x>)
+ %par_ty = acc.par_width %c4_pw par_dim(#acc.par_dim<thread_y>)
+ %par_tx = acc.par_width %c32_pw par_dim(#acc.par_dim<thread_x>)
+ acc.kernel_environment {
+ %gang_priv = acc.privatize par_dims(#acc<par_dims[block_x]>) : () -> !acc.private_type<memref<i64>>
+ %worker_priv = acc.privatize par_dims(#acc<par_dims[block_x, thread_y]>) : () -> !acc.private_type<memref<i64>>
+ acc.compute_region launch(%grid = %par_bx, %worker = %par_ty, %block = %par_tx)
+ ins(%arg_gang = %gang_priv, %arg_worker = %worker_priv)
+ : (!acc.private_type<memref<i64>>, !acc.private_type<memref<i64>>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c8 = arith.constant 8 : index
+ scf.parallel (%gang_iv) = (%c0) to (%grid) step (%c1) {
+ scf.parallel (%worker_iv) = (%c0) to (%worker) step (%c1) {
+ %plg = acc.private_local %arg_gang : (!acc.private_type<memref<i64>>) -> memref<i64>
+ %plw = acc.private_local %arg_worker : (!acc.private_type<memref<i64>>) -> memref<i64>
+ scf.parallel (%k) = (%c0) to (%c8) step (%c1) {
+ acc.predicate_region {
+ %val = arith.index_cast %k : index to i64
+ memref.store %val, %plg[] : memref<i64>
+ memref.store %val, %plw[] : memref<i64>
+ }
+ scf.parallel (%vec_iv) = (%c0) to (%block) step (%c1) {
+ %v = memref.load %plw[] : memref<i64>
+ memref.store %v, %plw[] : memref<i64>
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[thread_x]>}
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[sequential]>}
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[thread_y]>}
+ scf.reduce
+ } {acc.par_dims = #acc<par_dims[block_x]>}
+ acc.yield
+ } <{origin = "acc.parallel"}>
+ }
+ return
+}
More information about the Mlir-commits
mailing list