[llvm-branch-commits] [clang] [llvm] [OpenMP] Use device scope for atomic constructs on the device (PR #240426)
Robert Imschweiler via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 9 07:51:36 PST 2026
https://github.com/ro-i created https://github.com/llvm/llvm-project/pull/240426
OpenMP 6.0, 17.8.5 specifies that "The memscope clause determines the binding thread set for an atomic region. If the memscope clause is not present, the behavior is as if the memscope clause appeared on the construct with the device scope-specifier."
Thread the sync scopes through codegen amd also add a release note because this is changing existing behavior.
Existing behavior was overly cautios for OpenMP 5.2, too, though. The atomic construct never specified the option of having system-level scope. This only came with the memscope clause and it's "all" scope-specifier.
Claude assisted with this patch.
>From f3c12819ea98869b961148b12de04222f4106675 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 9 Nov 2026 04:02:07 -0600
Subject: [PATCH] [OpenMP] Use device scope for atomic constructs on the device
OpenMP 6.0, 17.8.5 specifies that "The memscope clause determines the
binding thread set for an atomic region. If the memscope clause is not
present, the behavior is as if the memscope clause appeared on the
construct with the device scope-specifier."
Thread the sync scopes through codegen amd also add a release note
because this is changing existing behavior.
Existing behavior was overly cautios for OpenMP 5.2, too, though. The
atomic construct never specified the option of having system-level
scope. This only came with the memscope clause and it's "all"
scope-specifier.
Claude assisted with this patch.
---
clang/docs/ReleaseNotes.md | 3 +
clang/lib/CodeGen/CGAtomic.cpp | 27 ++--
clang/lib/CodeGen/CGStmtOpenMP.cpp | 33 +++--
clang/lib/CodeGen/CodeGenFunction.h | 9 +-
.../test/OpenMP/amdgpu-unsafe-fp-atomics.cpp | 16 +--
.../OpenMP/nvptx_nested_parallel_codegen.cpp | 4 +-
.../OpenMP/target_atomic_device_scope.cpp | 130 ++++++++++++++++++
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 7 +-
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 21 +--
9 files changed, 205 insertions(+), 45 deletions(-)
create mode 100644 clang/test/OpenMP/target_atomic_device_scope.cpp
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index 603b3096259cb5..e6d1de482fcd54 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -1546,6 +1546,9 @@ The `alpha.cplusplus.UseAfterLifetimeEnd` checker was renamed to `alpha.core.Use
### OpenMP Support
+- `atomic` constructs in target device code now use device scope instead of
+ system scope, matching the OpenMP 6.0 default when no `memscope` clause is
+ given.
- Fixed an OpenMP `requires` directive read from a PCH or module losing its effect on
semantic checks, which caused spurious `reverse_offload` errors.
- Added the OpenMP 6.1 `#pragma omp flatten` loop transformation and the
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index cd7b278eab15d9..d10f381e71648b 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -38,10 +38,13 @@ namespace {
bool UseLibcall;
LValue LVal;
CGBitFieldInfo BFI;
+ llvm::SyncScope::ID Scope;
+
public:
- AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
+ AtomicInfo(CodeGenFunction &CGF, LValue &lvalue,
+ llvm::SyncScope::ID Scope = llvm::SyncScope::System)
: CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
- EvaluationKind(TEK_Scalar), UseLibcall(true) {
+ EvaluationKind(TEK_Scalar), UseLibcall(true), Scope(Scope) {
assert(!lvalue.isGlobalReg());
ASTContext &C = CGF.getContext();
if (lvalue.isSimple()) {
@@ -1631,7 +1634,7 @@ llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
if (shouldCastToInt(Addr.getElementType(), CmpXchg))
Addr = castToAtomicIntPointer(Addr);
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
- Load->setAtomic(AO);
+ Load->setAtomic(AO, Scope);
CGF.getTargetHooks().setTargetAtomicMetadata(CGF, *Load);
// Other decoration.
@@ -1705,8 +1708,9 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
/// we produce is an r-value of the atomic *value* type.
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
llvm::AtomicOrdering AO, bool IsVolatile,
- AggValueSlot resultSlot) {
- AtomicInfo Atomics(*this, src);
+ AggValueSlot resultSlot,
+ llvm::SyncScope::ID SSID) {
+ AtomicInfo Atomics(*this, src, SSID);
return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
IsVolatile);
}
@@ -1796,7 +1800,7 @@ std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
// Do the atomic store.
Address Addr = getAtomicAddressAsAtomicIntPointer();
auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
- Success, Failure);
+ Success, Failure, Scope);
// Other decoration.
Inst->setVolatile(LVal.isVolatileQualified());
Inst->setWeak(IsWeak);
@@ -2102,14 +2106,14 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
/// storage for any padding that was necessary.
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
llvm::AtomicOrdering AO, bool IsVolatile,
- bool isInit) {
+ bool isInit, llvm::SyncScope::ID SSID) {
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!rvalue.isAggregate() ||
rvalue.getAggregateAddress().getElementType() ==
dest.getAddress().getElementType());
- AtomicInfo atomics(*this, dest);
+ AtomicInfo atomics(*this, dest, SSID);
LValue LVal = atomics.getAtomicLValue();
// If this is an initialization, just put the value there normally.
@@ -2157,7 +2161,7 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
AO = llvm::AtomicOrdering::Release;
// Initializations don't need to be atomic.
if (!isInit) {
- store->setAtomic(AO);
+ store->setAtomic(AO, SSID);
getTargetHooks().setTargetAtomicMetadata(*this, *store);
}
@@ -2212,8 +2216,9 @@ llvm::FenceInst *CodeGenFunction::emitAtomicFence(llvm::AtomicOrdering Order,
void CodeGenFunction::EmitAtomicUpdate(
LValue LVal, llvm::AtomicOrdering AO,
- const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
- AtomicInfo Atomics(*this, LVal);
+ const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile,
+ llvm::SyncScope::ID SSID) {
+ AtomicInfo Atomics(*this, LVal, SSID);
Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
}
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 6b77bf53a29317..c940dee27f0e53 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -6926,12 +6926,23 @@ convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
return ComplexVal;
}
+/// Without a memscope clause, the binding thread set of an atomic region is
+/// all threads on the device.
+static llvm::SyncScope::ID getOMPAtomicSyncScope(CodeGenFunction &CGF,
+ llvm::AtomicOrdering AO) {
+ if (!CGF.getLangOpts().OpenMPIsTargetDevice)
+ return llvm::SyncScope::System;
+ return CGF.getTargetHooks().getLLVMSyncScopeID(
+ CGF.getLangOpts(), SyncScope::DeviceScope, AO, CGF.getLLVMContext());
+}
+
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
LValue LVal, RValue RVal) {
if (LVal.isGlobalReg())
CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
else
- CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);
+ CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false,
+ getOMPAtomicSyncScope(CGF, AO));
}
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
@@ -6939,9 +6950,11 @@ static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
SourceLocation Loc) {
if (LVal.isGlobalReg())
return CGF.EmitLoadOfLValue(LVal, Loc);
- return CGF.EmitAtomicLoad(
- LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
- LVal.isVolatile());
+ llvm::AtomicOrdering LoadAO =
+ llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
+ return CGF.EmitAtomicLoad(LVal, Loc, LoadAO, LVal.isVolatile(),
+ AggValueSlot::ignored(),
+ getOMPAtomicSyncScope(CGF, LoadAO));
}
void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
@@ -7133,8 +7146,8 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
X.getAddress().getElementType());
}
- llvm::AtomicRMWInst *Res =
- CGF.emitAtomicRMWInst(RMWOp, X.getAddress(), UpdateVal, AO);
+ llvm::AtomicRMWInst *Res = CGF.emitAtomicRMWInst(
+ RMWOp, X.getAddress(), UpdateVal, AO, getOMPAtomicSyncScope(CGF, AO));
return std::make_pair(true, RValue::get(Res));
}
@@ -7156,7 +7169,8 @@ std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
} else {
// Perform compare-and-swap procedure.
- EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
+ EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified(),
+ getOMPAtomicSyncScope(*this, AO));
}
}
return Res;
@@ -7409,18 +7423,19 @@ static void emitOMPAtomicCompareExpr(
bool FineGrainedMemory =
AtomicOpts.getOption(AtomicOptionKind::FineGrainedMemory);
bool RemoteMemory = AtomicOpts.getOption(AtomicOptionKind::RemoteMemory);
+ llvm::SyncScope::ID SSID = getOMPAtomicSyncScope(CGF, AO);
if (FailAO == llvm::AtomicOrdering::NotAtomic) {
// fail clause was not mentioned on the
// "#pragma omp atomic compare" construct.
CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
IsPostfixUpdate, IsFailOnly, /*IsWeak=*/false, IgnoreDenormalMode,
- FineGrainedMemory, RemoteMemory));
+ FineGrainedMemory, RemoteMemory, SSID));
} else
CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
IsPostfixUpdate, IsFailOnly, FailAO, /*IsWeak=*/false,
- IgnoreDenormalMode, FineGrainedMemory, RemoteMemory));
+ IgnoreDenormalMode, FineGrainedMemory, RemoteMemory, SSID));
}
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index e6ecf3bca3e94c..0152fd44327010 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4440,12 +4440,14 @@ class CodeGenFunction : public CodeGenTypeCache {
RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
llvm::AtomicOrdering AO, bool IsVolatile = false,
- AggValueSlot slot = AggValueSlot::ignored());
+ AggValueSlot slot = AggValueSlot::ignored(),
+ llvm::SyncScope::ID SSID = llvm::SyncScope::System);
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
- bool IsVolatile, bool isInit);
+ bool IsVolatile, bool isInit,
+ llvm::SyncScope::ID SSID = llvm::SyncScope::System);
std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
@@ -4471,7 +4473,8 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
const llvm::function_ref<RValue(RValue)> &UpdateOp,
- bool IsVolatile);
+ bool IsVolatile,
+ llvm::SyncScope::ID SSID = llvm::SyncScope::System);
/// EmitToMemory - Change a scalar value from its value
/// representation to its in-memory representation.
diff --git a/clang/test/OpenMP/amdgpu-unsafe-fp-atomics.cpp b/clang/test/OpenMP/amdgpu-unsafe-fp-atomics.cpp
index f1771252a4d58c..59f59419c48fe8 100644
--- a/clang/test/OpenMP/amdgpu-unsafe-fp-atomics.cpp
+++ b/clang/test/OpenMP/amdgpu-unsafe-fp-atomics.cpp
@@ -11,7 +11,7 @@ double dv, dx;
// DEFAULT-SAME: ) #[[ATTR0:[0-9]+]] {
// DEFAULT-NEXT: [[ENTRY:.*:]]
// DEFAULT-NEXT: [[TMP0:%.*]] = load float, ptr addrspacecast (ptr addrspace(1) @fv to ptr), align 4
-// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4:![0-9]+]], !amdgpu.no.remote.memory [[META4]]
+// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4:![0-9]+]], !amdgpu.no.remote.memory [[META4]]
// DEFAULT-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP0]]
// DEFAULT-NEXT: store float [[ADD]], ptr addrspacecast (ptr addrspace(1) @fv to ptr), align 4
// DEFAULT-NEXT: ret void
@@ -20,7 +20,7 @@ double dv, dx;
// UNSAFE-FP-ATOMICS-SAME: ) #[[ATTR0:[0-9]+]] {
// UNSAFE-FP-ATOMICS-NEXT: [[ENTRY:.*:]]
// UNSAFE-FP-ATOMICS-NEXT: [[TMP0:%.*]] = load float, ptr addrspacecast (ptr addrspace(1) @fv to ptr), align 4
-// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] monotonic, align 4, !atomic.ignore.denormal.mode [[META4:![0-9]+]], !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] syncscope("agent") monotonic, align 4, !atomic.ignore.denormal.mode [[META4:![0-9]+]], !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
// UNSAFE-FP-ATOMICS-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP0]]
// UNSAFE-FP-ATOMICS-NEXT: store float [[ADD]], ptr addrspacecast (ptr addrspace(1) @fv to ptr), align 4
// UNSAFE-FP-ATOMICS-NEXT: ret void
@@ -34,7 +34,7 @@ void atomic_fadd_f32() {
// DEFAULT-SAME: ) #[[ATTR0]] {
// DEFAULT-NEXT: [[ENTRY:.*:]]
// DEFAULT-NEXT: [[TMP0:%.*]] = load double, ptr addrspacecast (ptr addrspace(1) @dv to ptr), align 8
-// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
// DEFAULT-NEXT: [[ADD:%.*]] = fadd double [[TMP1]], [[TMP0]]
// DEFAULT-NEXT: store double [[ADD]], ptr addrspacecast (ptr addrspace(1) @dv to ptr), align 8
// DEFAULT-NEXT: ret void
@@ -43,7 +43,7 @@ void atomic_fadd_f32() {
// UNSAFE-FP-ATOMICS-SAME: ) #[[ATTR0]] {
// UNSAFE-FP-ATOMICS-NEXT: [[ENTRY:.*:]]
// UNSAFE-FP-ATOMICS-NEXT: [[TMP0:%.*]] = load double, ptr addrspacecast (ptr addrspace(1) @dv to ptr), align 8
-// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
// UNSAFE-FP-ATOMICS-NEXT: [[ADD:%.*]] = fadd double [[TMP1]], [[TMP0]]
// UNSAFE-FP-ATOMICS-NEXT: store double [[ADD]], ptr addrspacecast (ptr addrspace(1) @dv to ptr), align 8
// UNSAFE-FP-ATOMICS-NEXT: ret void
@@ -57,14 +57,14 @@ void atomic_fadd_f64() {
// DEFAULT-SAME: ) #[[ATTR0]] {
// DEFAULT-NEXT: [[ENTRY:.*:]]
// DEFAULT-NEXT: [[TMP0:%.*]] = load double, ptr addrspacecast (ptr addrspace(1) @dv to ptr), align 8
-// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fmin ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fmin ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
// DEFAULT-NEXT: ret void
//
// UNSAFE-FP-ATOMICS-LABEL: define hidden void @_Z23atomic_compare_fmin_f64v(
// UNSAFE-FP-ATOMICS-SAME: ) #[[ATTR0]] {
// UNSAFE-FP-ATOMICS-NEXT: [[ENTRY:.*:]]
// UNSAFE-FP-ATOMICS-NEXT: [[TMP0:%.*]] = load double, ptr addrspacecast (ptr addrspace(1) @dv to ptr), align 8
-// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fmin ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] monotonic, align 8, !atomic.ignore.denormal.mode [[META4]], !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fmin ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP0]] syncscope("agent") monotonic, align 8, !atomic.ignore.denormal.mode [[META4]], !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
// UNSAFE-FP-ATOMICS-NEXT: ret void
//
void atomic_compare_fmin_f64() {
@@ -77,14 +77,14 @@ void atomic_compare_fmin_f64() {
// DEFAULT-SAME: ) #[[ATTR0]] {
// DEFAULT-NEXT: [[ENTRY:.*:]]
// DEFAULT-NEXT: [[TMP0:%.*]] = load float, ptr addrspacecast (ptr addrspace(1) @fv to ptr), align 4
-// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fmax ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// DEFAULT-NEXT: [[TMP1:%.*]] = atomicrmw fmax ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
// DEFAULT-NEXT: ret void
//
// UNSAFE-FP-ATOMICS-LABEL: define hidden void @_Z23atomic_compare_fmax_f32v(
// UNSAFE-FP-ATOMICS-SAME: ) #[[ATTR0]] {
// UNSAFE-FP-ATOMICS-NEXT: [[ENTRY:.*:]]
// UNSAFE-FP-ATOMICS-NEXT: [[TMP0:%.*]] = load float, ptr addrspacecast (ptr addrspace(1) @fv to ptr), align 4
-// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fmax ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] monotonic, align 4, !atomic.ignore.denormal.mode [[META4]], !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// UNSAFE-FP-ATOMICS-NEXT: [[TMP1:%.*]] = atomicrmw fmax ptr addrspacecast (ptr addrspace(1) @fx to ptr), float [[TMP0]] syncscope("agent") monotonic, align 4, !atomic.ignore.denormal.mode [[META4]], !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
// UNSAFE-FP-ATOMICS-NEXT: ret void
//
void atomic_compare_fmax_f32() {
diff --git a/clang/test/OpenMP/nvptx_nested_parallel_codegen.cpp b/clang/test/OpenMP/nvptx_nested_parallel_codegen.cpp
index da833bd2f94552..39f98bed8a6392 100644
--- a/clang/test/OpenMP/nvptx_nested_parallel_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_nested_parallel_codegen.cpp
@@ -123,7 +123,7 @@ int main() {
// CHECK1-NEXT: [[C_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: store ptr [[C]], ptr [[C_ADDR]], align 8
// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[TMP0]], i32 1 monotonic, align 4
+// CHECK1-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[TMP0]], i32 1 syncscope("device") monotonic, align 4
// CHECK1-NEXT: ret void
//
//
@@ -235,7 +235,7 @@ int main() {
// CHECK2-NEXT: [[C_ADDR:%.*]] = alloca ptr, align 4
// CHECK2-NEXT: store ptr [[C]], ptr [[C_ADDR]], align 4
// CHECK2-NEXT: [[TMP0:%.*]] = load ptr, ptr [[C_ADDR]], align 4
-// CHECK2-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[TMP0]], i32 1 monotonic, align 4
+// CHECK2-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[TMP0]], i32 1 syncscope("device") monotonic, align 4
// CHECK2-NEXT: ret void
//
//
diff --git a/clang/test/OpenMP/target_atomic_device_scope.cpp b/clang/test/OpenMP/target_atomic_device_scope.cpp
new file mode 100644
index 00000000000000..b16c5c447f7756
--- /dev/null
+++ b/clang/test/OpenMP/target_atomic_device_scope.cpp
@@ -0,0 +1,130 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck -check-prefix=HOST %s
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple amdgcn-amd-amdhsa -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm %s -fopenmp-is-target-device -o - | FileCheck -check-prefix=AMDGPU %s
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple nvptx64-nvidia-cuda -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-target-device -o - | FileCheck -check-prefix=NVPTX %s
+
+#pragma omp declare target
+
+int iv, ix;
+float fv, fx;
+double dv, dx;
+
+// HOST-LABEL: define dso_local void @_Z10atomic_opsv(
+// HOST-SAME: ) #[[ATTR0:[0-9]+]] {
+// HOST-NEXT: [[ENTRY:.*]]:
+// HOST-NEXT: [[ATOMIC_TEMP:%.*]] = alloca float, align 4
+// HOST-NEXT: [[ATOMIC_LOAD:%.*]] = load atomic i32, ptr @ix monotonic, align 4
+// HOST-NEXT: store i32 [[ATOMIC_LOAD]], ptr @iv, align 4
+// HOST-NEXT: [[TMP0:%.*]] = load i32, ptr @iv, align 4
+// HOST-NEXT: store atomic i32 [[TMP0]], ptr @ix monotonic, align 4
+// HOST-NEXT: [[TMP1:%.*]] = load i32, ptr @iv, align 4
+// HOST-NEXT: [[TMP2:%.*]] = atomicrmw add ptr @ix, i32 [[TMP1]] monotonic, align 4
+// HOST-NEXT: [[TMP3:%.*]] = load float, ptr @fv, align 4
+// HOST-NEXT: [[ATOMIC_LOAD1:%.*]] = load atomic i32, ptr @fx monotonic, align 4
+// HOST-NEXT: br label %[[ATOMIC_CONT:.*]]
+// HOST: [[ATOMIC_CONT]]:
+// HOST-NEXT: [[TMP4:%.*]] = phi i32 [ [[ATOMIC_LOAD1]], %[[ENTRY]] ], [ [[TMP8:%.*]], %[[ATOMIC_CONT]] ]
+// HOST-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
+// HOST-NEXT: [[MUL:%.*]] = fmul float [[TMP5]], [[TMP3]]
+// HOST-NEXT: store float [[MUL]], ptr [[ATOMIC_TEMP]], align 4
+// HOST-NEXT: [[TMP6:%.*]] = load i32, ptr [[ATOMIC_TEMP]], align 4
+// HOST-NEXT: [[TMP7:%.*]] = cmpxchg ptr @fx, i32 [[TMP4]], i32 [[TMP6]] monotonic monotonic, align 4
+// HOST-NEXT: [[TMP8]] = extractvalue { i32, i1 } [[TMP7]], 0
+// HOST-NEXT: [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+// HOST-NEXT: br i1 [[TMP9]], label %[[ATOMIC_EXIT:.*]], label %[[ATOMIC_CONT]]
+// HOST: [[ATOMIC_EXIT]]:
+// HOST-NEXT: [[TMP10:%.*]] = atomicrmw add ptr @ix, i32 1 seq_cst, align 4
+// HOST-NEXT: store i32 [[TMP10]], ptr @iv, align 4
+// HOST-NEXT: [[TMP11:%.*]] = load double, ptr @dv, align 8
+// HOST-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr @dx, double [[TMP11]] monotonic, align 8
+// HOST-NEXT: [[TMP13:%.*]] = load i32, ptr @iv, align 4
+// HOST-NEXT: [[TMP14:%.*]] = cmpxchg ptr @ix, i32 [[TMP13]], i32 0 monotonic monotonic, align 4
+// HOST-NEXT: ret void
+//
+// AMDGPU-LABEL: define hidden void @_Z10atomic_opsv(
+// AMDGPU-SAME: ) #[[ATTR0:[0-9]+]] {
+// AMDGPU-NEXT: [[ENTRY:.*]]:
+// AMDGPU-NEXT: [[ATOMIC_TEMP:%.*]] = alloca float, align 4, addrspace(5)
+// AMDGPU-NEXT: [[ATOMIC_LOAD:%.*]] = load atomic i32, ptr addrspacecast (ptr addrspace(1) @ix to ptr) syncscope("agent") monotonic, align 4
+// AMDGPU-NEXT: store i32 [[ATOMIC_LOAD]], ptr addrspacecast (ptr addrspace(1) @iv to ptr), align 4
+// AMDGPU-NEXT: [[TMP0:%.*]] = load i32, ptr addrspacecast (ptr addrspace(1) @iv to ptr), align 4
+// AMDGPU-NEXT: store atomic i32 [[TMP0]], ptr addrspacecast (ptr addrspace(1) @ix to ptr) syncscope("agent") monotonic, align 4
+// AMDGPU-NEXT: [[TMP1:%.*]] = load i32, ptr addrspacecast (ptr addrspace(1) @iv to ptr), align 4
+// AMDGPU-NEXT: [[TMP2:%.*]] = atomicrmw add ptr addrspacecast (ptr addrspace(1) @ix to ptr), i32 [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4:![0-9]+]], !amdgpu.no.remote.memory [[META4]]
+// AMDGPU-NEXT: [[TMP3:%.*]] = load float, ptr addrspacecast (ptr addrspace(1) @fv to ptr), align 4
+// AMDGPU-NEXT: [[ATOMIC_LOAD1:%.*]] = load atomic i32, ptr addrspacecast (ptr addrspace(1) @fx to ptr) syncscope("agent") monotonic, align 4
+// AMDGPU-NEXT: br label %[[ATOMIC_CONT:.*]]
+// AMDGPU: [[ATOMIC_CONT]]:
+// AMDGPU-NEXT: [[TMP4:%.*]] = phi i32 [ [[ATOMIC_LOAD1]], %[[ENTRY]] ], [ [[TMP8:%.*]], %[[ATOMIC_CONT]] ]
+// AMDGPU-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
+// AMDGPU-NEXT: [[MUL:%.*]] = fmul float [[TMP5]], [[TMP3]]
+// AMDGPU-NEXT: store float [[MUL]], ptr addrspace(5) [[ATOMIC_TEMP]], align 4
+// AMDGPU-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(5) [[ATOMIC_TEMP]], align 4
+// AMDGPU-NEXT: [[TMP7:%.*]] = cmpxchg ptr addrspacecast (ptr addrspace(1) @fx to ptr), i32 [[TMP4]], i32 [[TMP6]] syncscope("agent") monotonic monotonic, align 4
+// AMDGPU-NEXT: [[TMP8]] = extractvalue { i32, i1 } [[TMP7]], 0
+// AMDGPU-NEXT: [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+// AMDGPU-NEXT: br i1 [[TMP9]], label %[[ATOMIC_EXIT:.*]], label %[[ATOMIC_CONT]]
+// AMDGPU: [[ATOMIC_EXIT]]:
+// AMDGPU-NEXT: [[TMP10:%.*]] = atomicrmw add ptr addrspacecast (ptr addrspace(1) @ix to ptr), i32 1 syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// AMDGPU-NEXT: store i32 [[TMP10]], ptr addrspacecast (ptr addrspace(1) @iv to ptr), align 4
+// AMDGPU-NEXT: [[TMP11:%.*]] = load double, ptr addrspacecast (ptr addrspace(1) @dv to ptr), align 8
+// AMDGPU-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspacecast (ptr addrspace(1) @dx to ptr), double [[TMP11]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.no.remote.memory [[META4]]
+// AMDGPU-NEXT: [[TMP13:%.*]] = load i32, ptr addrspacecast (ptr addrspace(1) @iv to ptr), align 4
+// AMDGPU-NEXT: [[TMP14:%.*]] = cmpxchg ptr addrspacecast (ptr addrspace(1) @ix to ptr), i32 [[TMP13]], i32 0 syncscope("agent") monotonic monotonic, align 4
+// AMDGPU-NEXT: ret void
+//
+// NVPTX-LABEL: define hidden void @_Z10atomic_opsv(
+// NVPTX-SAME: ) #[[ATTR0:[0-9]+]] {
+// NVPTX-NEXT: [[ENTRY:.*]]:
+// NVPTX-NEXT: [[ATOMIC_TEMP:%.*]] = alloca float, align 4
+// NVPTX-NEXT: [[ATOMIC_LOAD:%.*]] = load atomic i32, ptr @ix syncscope("device") monotonic, align 4
+// NVPTX-NEXT: store i32 [[ATOMIC_LOAD]], ptr @iv, align 4
+// NVPTX-NEXT: [[TMP0:%.*]] = load i32, ptr @iv, align 4
+// NVPTX-NEXT: store atomic i32 [[TMP0]], ptr @ix syncscope("device") monotonic, align 4
+// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr @iv, align 4
+// NVPTX-NEXT: [[TMP2:%.*]] = atomicrmw add ptr @ix, i32 [[TMP1]] syncscope("device") monotonic, align 4
+// NVPTX-NEXT: [[TMP3:%.*]] = load float, ptr @fv, align 4
+// NVPTX-NEXT: [[ATOMIC_LOAD1:%.*]] = load atomic i32, ptr @fx syncscope("device") monotonic, align 4
+// NVPTX-NEXT: br label %[[ATOMIC_CONT:.*]]
+// NVPTX: [[ATOMIC_CONT]]:
+// NVPTX-NEXT: [[TMP4:%.*]] = phi i32 [ [[ATOMIC_LOAD1]], %[[ENTRY]] ], [ [[TMP8:%.*]], %[[ATOMIC_CONT]] ]
+// NVPTX-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
+// NVPTX-NEXT: [[MUL:%.*]] = fmul float [[TMP5]], [[TMP3]]
+// NVPTX-NEXT: store float [[MUL]], ptr [[ATOMIC_TEMP]], align 4
+// NVPTX-NEXT: [[TMP6:%.*]] = load i32, ptr [[ATOMIC_TEMP]], align 4
+// NVPTX-NEXT: [[TMP7:%.*]] = cmpxchg ptr @fx, i32 [[TMP4]], i32 [[TMP6]] syncscope("device") monotonic monotonic, align 4
+// NVPTX-NEXT: [[TMP8]] = extractvalue { i32, i1 } [[TMP7]], 0
+// NVPTX-NEXT: [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+// NVPTX-NEXT: br i1 [[TMP9]], label %[[ATOMIC_EXIT:.*]], label %[[ATOMIC_CONT]]
+// NVPTX: [[ATOMIC_EXIT]]:
+// NVPTX-NEXT: [[TMP10:%.*]] = atomicrmw add ptr @ix, i32 1 syncscope("device") seq_cst, align 4
+// NVPTX-NEXT: store i32 [[TMP10]], ptr @iv, align 4
+// NVPTX-NEXT: [[TMP11:%.*]] = load double, ptr @dv, align 8
+// NVPTX-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr @dx, double [[TMP11]] syncscope("device") monotonic, align 8
+// NVPTX-NEXT: [[TMP13:%.*]] = load i32, ptr @iv, align 4
+// NVPTX-NEXT: [[TMP14:%.*]] = cmpxchg ptr @ix, i32 [[TMP13]], i32 0 syncscope("device") monotonic monotonic, align 4
+// NVPTX-NEXT: ret void
+//
+void atomic_ops() {
+#pragma omp atomic read
+ iv = ix;
+#pragma omp atomic write
+ ix = iv;
+#pragma omp atomic update
+ ix += iv;
+#pragma omp atomic update
+ fx *= fv;
+#pragma omp atomic capture seq_cst
+ iv = ix++;
+#pragma omp atomic compare
+ if (dv < dx)
+ dx = dv;
+#pragma omp atomic compare
+ if (ix == iv)
+ ix = 0;
+}
+
+#pragma omp end declare target
+//.
+// AMDGPU: [[META4]] = !{}
+//.
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index a3518327bb6879..275c14240f0b8e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -4276,6 +4276,8 @@ class OpenMPIRBuilder {
/// memory.
/// \param IsRemoteMemory False if X is known not to be in remote
/// memory.
+ /// \param SSID Synchronization scope of the generated atomic
+ /// instructions.
///
/// \return Insertion point after generated atomic capture IR.
/// Whether to emit special handling for IEEE 754 -0.0 == +0.0 in
@@ -4295,14 +4297,15 @@ class OpenMPIRBuilder {
AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
bool IsFailOnly, bool IsWeak = false, bool IsIgnoreDenormalMode = false,
- bool IsFineGrainedMemory = true, bool IsRemoteMemory = true);
+ bool IsFineGrainedMemory = true, bool IsRemoteMemory = true,
+ SyncScope::ID SSID = SyncScope::System);
LLVM_ABI InsertPointTy createAtomicCompare(
const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
bool IsFailOnly, AtomicOrdering Failure, bool IsWeak = false,
bool IsIgnoreDenormalMode = false, bool IsFineGrainedMemory = true,
- bool IsRemoteMemory = true);
+ bool IsRemoteMemory = true, SyncScope::ID SSID = SyncScope::System);
/// Create the control flow structure of a canonical OpenMP loop.
///
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 57127bc6e3818c..fce2e07d702df2 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -12010,13 +12010,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
bool IsFailOnly, bool IsWeak, bool IsIgnoreDenormalMode,
- bool IsFineGrainedMemory, bool IsRemoteMemory) {
+ bool IsFineGrainedMemory, bool IsRemoteMemory, SyncScope::ID SSID) {
AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
IsPostfixUpdate, IsFailOnly, Failure, IsWeak,
IsIgnoreDenormalMode, IsFineGrainedMemory,
- IsRemoteMemory);
+ IsRemoteMemory, SSID);
}
OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
@@ -12024,7 +12024,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
bool IsFailOnly, AtomicOrdering Failure, bool IsWeak,
- bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
+ bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory,
+ SyncScope::ID SSID) {
if (!updateToLocation(Loc))
return Loc.IP;
@@ -12082,7 +12083,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
// Load X atomically.
LoadInst *XCurr = Builder.CreateLoad(IntCastTy, X.Var,
X.Var->getName() + ".atomic.load");
- XCurr->setAtomic(AtomicOrdering::Monotonic);
+ XCurr->setAtomic(AtomicOrdering::Monotonic, SSID);
Value *XFP = Builder.CreateBitCast(XCurr, X.ElemTy);
// IEEE 754: NaN != NaN, but cmpxchg would succeed if E and X have
@@ -12128,7 +12129,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
// ZeroBB: cmpxchg with X's loaded bit-pattern.
Builder.SetInsertPoint(ZeroBB);
AtomicCmpXchgInst *ResZero = Builder.CreateAtomicCmpXchg(
- X.Var, XCurr, DBCast, MaybeAlign(), AO, Failure);
+ X.Var, XCurr, DBCast, MaybeAlign(), AO, Failure, SSID);
ResZero->setWeak(IsWeak);
Value *OldZero = Builder.CreateExtractValue(ResZero, /*Idxs=*/0);
Value *OkZero = Builder.CreateExtractValue(ResZero, /*Idxs=*/1);
@@ -12137,7 +12138,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
// NormalBB: original bitwise cmpxchg.
Builder.SetInsertPoint(NormalBB);
AtomicCmpXchgInst *ResNormal = Builder.CreateAtomicCmpXchg(
- X.Var, EBCast, DBCast, MaybeAlign(), AO, Failure);
+ X.Var, EBCast, DBCast, MaybeAlign(), AO, Failure, SSID);
ResNormal->setWeak(IsWeak);
Value *OldNormal = Builder.CreateExtractValue(ResNormal, /*Idxs=*/0);
Value *OkNormal = Builder.CreateExtractValue(ResNormal, /*Idxs=*/1);
@@ -12174,10 +12175,10 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
Value *EBCast = Builder.CreateBitCast(E, IntCastTy);
Value *DBCast = Builder.CreateBitCast(D, IntCastTy);
Result = Builder.CreateAtomicCmpXchg(X.Var, EBCast, DBCast,
- MaybeAlign(), AO, Failure);
+ MaybeAlign(), AO, Failure, SSID);
} else {
- Result =
- Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure);
+ Result = Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO,
+ Failure, SSID);
}
Result->setWeak(IsWeak);
@@ -12330,7 +12331,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
}
AtomicRMWInst *OldValue =
- Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO);
+ Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO, SSID);
setAtomicRMWMetadata(OldValue, T, IsIgnoreDenormalMode, IsFineGrainedMemory,
IsRemoteMemory);
if (V.Var) {
More information about the llvm-branch-commits
mailing list