[clang] [CIR][AMDGPU][SPIR-V] Emit AMDGPU atomic memory metadata for spirv64-amd (PR #236037)

Arseniy Obolenskiy via cfe-commits cfe-commits at lists.llvm.org
Tue Oct 27 01:38:37 PDT 2026


https://github.com/aobolensk created https://github.com/llvm/llvm-project/pull/236037

Match classic codegen so AMD SPIR-V atomic RMW ops do not lower to CAS loops

>From 4b77044464f5bf197d7c419fb2e52c1d510dceab Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 27 Oct 2026 09:35:55 +0100
Subject: [PATCH] [CIR][AMDGPU][SPIR-V] Emit AMDGPU atomic memory metadata for
 spirv64-amd

Match classic codegen so AMD SPIR-V atomic RMW ops do not lower to CAS loops
---
 clang/lib/CIR/CodeGen/CIRGenAtomic.cpp        | 14 ++++---
 .../CodeGenHIP/amdgcnspirv-atomic-scope.hip   | 40 ++++++++++++-------
 2 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
index c4ebf6f0afcec2..bc4aa79b873ead 100644
--- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
@@ -171,9 +171,15 @@ static void setAMDGPUAtomicMetadata(CIRGenFunction &cgf, mlir::Operation *op,
   // CIRGenStmt.cpp's emitAttributedStmt currently rejects via errorNYI.
   assert(!cir::MissingFeatures::atomicAMDGPUAvailableVisibleMMRA());
 
+  const llvm::Triple &triple = cgf.cgm.getTriple();
+  bool isAMDGCN = triple.isAMDGCN();
+  bool isAMDSPIRV = triple.isSPIRV() && triple.getVendor() == llvm::Triple::AMD;
+  if (!isAMDGCN && !isAMDSPIRV)
+    return;
+
   // Lowering keeps this only on a flat pointer.
   mlir::UnitAttr unit = cgf.getBuilder().getUnitAttr();
-  if (expr->threadPrivateMemoryAtomicsAreUndefined())
+  if (isAMDGCN && expr->threadPrivateMemoryAtomicsAreUndefined())
     op->setAttr(cir::CIRDialect::getAMDGPUNoPrivateAddrSpaceAttrName(), unit);
 
   // Only a read-modify-write instruction carries these; a plain load, store or
@@ -541,8 +547,7 @@ static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,
 
   cmpxchg.setIsVolatile(e->isVolatile());
   cmpxchg.setWeak(isWeak);
-  if (cgf.cgm.getTriple().isAMDGCN())
-    setAMDGPUAtomicMetadata(cgf, cmpxchg, e);
+  setAMDGPUAtomicMetadata(cgf, cmpxchg, e);
 
   mlir::Value failed = builder.createNot(cmpxchg.getSuccess());
   cir::IfOp::create(
@@ -971,8 +976,7 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
   if (fetchFirst && opName == cir::AtomicFetchOp::getOperationName())
     rmwOp->setAttr("fetch_first", builder.getUnitAttr());
 
-  if (cgf.cgm.getTriple().isAMDGCN())
-    setAMDGPUAtomicMetadata(cgf, rmwOp, expr);
+  setAMDGPUAtomicMetadata(cgf, rmwOp, expr);
 
   mlir::Value result = rmwOp->getResult(0);
 
diff --git a/clang/test/CIR/CodeGenHIP/amdgcnspirv-atomic-scope.hip b/clang/test/CIR/CodeGenHIP/amdgcnspirv-atomic-scope.hip
index b9339e3e0f67ad..a8096d14cb2b70 100644
--- a/clang/test/CIR/CodeGenHIP/amdgcnspirv-atomic-scope.hip
+++ b/clang/test/CIR/CodeGenHIP/amdgcnspirv-atomic-scope.hip
@@ -18,56 +18,66 @@
 #define __device__ __attribute__((device))
 
 // CIR-LABEL: @_Z12singlethreadPff
-// CIR: cir.atomic.fetch add relaxed syncscope(hip_single_thread)
+// CIR: cir.atomic.fetch add relaxed syncscope(hip_single_thread) {{.*}} {cir.amdgpu_no_fine_grained_memory, cir.amdgpu_no_remote_memory} loc(
 // LLVM-LABEL: @_Z12singlethreadPff
-// LLVM: atomicrmw fadd {{.*}} syncscope("singlethread") monotonic
+// LLVM: atomicrmw fadd {{.*}} syncscope("singlethread") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.no.remote.memory !{{[0-9]+}}{{$}}
 // SPIR-LABEL: @_Z12singlethreadPff
-// SPIR: cir.atomic.fetch add relaxed syncscope(system)
+// SPIR: cir.atomic.fetch add relaxed syncscope(system) {{.*}} -> !{{cir.float|s32i}} loc(
 __device__ float singlethread(float *p, float v) {
   return __hip_atomic_fetch_add(p, v, __ATOMIC_RELAXED,
                                 __HIP_MEMORY_SCOPE_SINGLETHREAD);
 }
 
 // CIR-LABEL: @_Z9wavefrontPii
-// CIR: cir.atomic.fetch add relaxed syncscope(hip_wavefront)
+// CIR: cir.atomic.fetch add relaxed syncscope(hip_wavefront) {{.*}} {cir.amdgpu_no_fine_grained_memory, cir.amdgpu_no_remote_memory} loc(
 // LLVM-LABEL: @_Z9wavefrontPii
-// LLVM: atomicrmw add {{.*}} syncscope("subgroup") monotonic
+// LLVM: atomicrmw add {{.*}} syncscope("subgroup") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.no.remote.memory !{{[0-9]+}}{{$}}
 // SPIR-LABEL: @_Z9wavefrontPii
-// SPIR: cir.atomic.fetch add relaxed syncscope(system)
+// SPIR: cir.atomic.fetch add relaxed syncscope(system) {{.*}} -> !{{cir.float|s32i}} loc(
 __device__ int wavefront(int *p, int v) {
   return __hip_atomic_fetch_add(p, v, __ATOMIC_RELAXED,
                                 __HIP_MEMORY_SCOPE_WAVEFRONT);
 }
 
 // CIR-LABEL: @_Z9workgroupPff
-// CIR: cir.atomic.fetch add relaxed syncscope(hip_workgroup)
+// CIR: cir.atomic.fetch add relaxed syncscope(hip_workgroup) {{.*}} {cir.amdgpu_no_fine_grained_memory, cir.amdgpu_no_remote_memory} loc(
 // LLVM-LABEL: @_Z9workgroupPff
-// LLVM: atomicrmw fadd {{.*}} syncscope("workgroup") monotonic
+// LLVM: atomicrmw fadd {{.*}} syncscope("workgroup") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.no.remote.memory !{{[0-9]+}}{{$}}
 // SPIR-LABEL: @_Z9workgroupPff
-// SPIR: cir.atomic.fetch add relaxed syncscope(system)
+// SPIR: cir.atomic.fetch add relaxed syncscope(system) {{.*}} -> !{{cir.float|s32i}} loc(
 __device__ float workgroup(float *p, float v) {
   return __hip_atomic_fetch_add(p, v, __ATOMIC_RELAXED,
                                 __HIP_MEMORY_SCOPE_WORKGROUP);
 }
 
 // CIR-LABEL: @_Z5agentPii
-// CIR: cir.atomic.fetch add relaxed syncscope(hip_agent)
+// CIR: cir.atomic.fetch add relaxed syncscope(hip_agent) {{.*}} {cir.amdgpu_no_fine_grained_memory, cir.amdgpu_no_remote_memory} loc(
 // LLVM-LABEL: @_Z5agentPii
-// LLVM: atomicrmw add {{.*}} syncscope("device") monotonic
+// LLVM: atomicrmw add {{.*}} syncscope("device") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.no.remote.memory !{{[0-9]+}}{{$}}
 // SPIR-LABEL: @_Z5agentPii
-// SPIR: cir.atomic.fetch add relaxed syncscope(system)
+// SPIR: cir.atomic.fetch add relaxed syncscope(system) {{.*}} -> !{{cir.float|s32i}} loc(
 __device__ int agent(int *p, int v) {
   return __hip_atomic_fetch_add(p, v, __ATOMIC_RELAXED,
                                 __HIP_MEMORY_SCOPE_AGENT);
 }
 
 // CIR-LABEL: @_Z6systemPff
-// CIR: cir.atomic.fetch add relaxed syncscope(hip_system)
+// CIR: cir.atomic.fetch add relaxed syncscope(hip_system) {{.*}} {cir.amdgpu_no_fine_grained_memory, cir.amdgpu_no_remote_memory} loc(
 // LLVM-LABEL: @_Z6systemPff
-// LLVM: atomicrmw fadd {{.*}} monotonic
+// LLVM: atomicrmw fadd {{.*}} monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.no.remote.memory !{{[0-9]+}}{{$}}
 // SPIR-LABEL: @_Z6systemPff
-// SPIR: cir.atomic.fetch add relaxed syncscope(system)
+// SPIR: cir.atomic.fetch add relaxed syncscope(system) {{.*}} -> !{{cir.float|s32i}} loc(
 __device__ float system(float *p, float v) {
   return __hip_atomic_fetch_add(p, v, __ATOMIC_RELAXED,
                                 __HIP_MEMORY_SCOPE_SYSTEM);
 }
+
+// CIR-LABEL: @_Z7cmpxchgPiii
+// CIR: cir.atomic.cmpxchg {{.*}} syncscope(hip_agent) {{.*}} -> (!s32i, !cir.bool) loc(
+// LLVM-LABEL: @_Z7cmpxchgPiii
+// LLVM: cmpxchg {{.*}} syncscope("device") monotonic monotonic, align 4{{$}}
+__device__ bool cmpxchg(int *p, int e, int d) {
+  return __hip_atomic_compare_exchange_strong(p, &e, d, __ATOMIC_RELAXED,
+                                              __ATOMIC_RELAXED,
+                                              __HIP_MEMORY_SCOPE_AGENT);
+}



More information about the cfe-commits mailing list