[clang] [Clang] Introduce scoped variants of GNU atomic functions (PR #72280)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Nov 14 08:16:43 PST 2023
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang-codegen
Author: Joseph Huber (jhuber6)
<details>
<summary>Changes</summary>
Summary:
The standard GNU atomic operations are a very common way to target
hardware atomics on the device. With more hetergenous devices being
introduced, the concept of memory scopes has been in the LLVM language
for awhile via the `syncscope` modifier. For targets, such as the GPU,
this can change code generation depending on whether or not we only need
to be consistent with the memory ordering with the entire system, the
single GPU device, or lower.
Previously these scopes were only exported via the `opencl` and `hip`
variants of these functions. However, this made it difficult to use
outside of those languages and the semantics were different from the
standard GNU versions. This patch introduces a `__scoped_atomic` variant
for the common functions. There was some discussion over whether or not
these should be overloads of the existing ones, or simply new variants.
I leant towards new variants to be less disruptive.
The scope here can be one of the following
```
__MEMORY_SCOPE_SYSTEM // All devices and systems
__MEMORY_SCOPE_DEVICE // Just this device
__MEMORY_SCOPE_WRKGRP // A 'work-group' AKA CUDA block
__MEMORY_SCOPE_WVFRNT // A 'wavefront' AKA CUDA warp
__MEMORY_SCOPE_SINGLE // A single thread.
```
Naming consistency was attempted, but it is difficult to cpature to full
spectrum with no many names. Suggestions appreciated.
---
Patch is 73.26 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/72280.diff
14 Files Affected:
- (modified) clang/include/clang/AST/Expr.h (+11-9)
- (modified) clang/include/clang/Basic/Builtins.def (+26)
- (modified) clang/include/clang/Basic/SyncScope.h (+68-1)
- (modified) clang/lib/AST/Expr.cpp (+26)
- (modified) clang/lib/AST/StmtPrinter.cpp (+1)
- (modified) clang/lib/CodeGen/CGAtomic.cpp (+116-9)
- (modified) clang/lib/CodeGen/Targets/AMDGPU.cpp (+5)
- (modified) clang/lib/Frontend/InitPreprocessor.cpp (+7)
- (modified) clang/lib/Sema/SemaChecking.cpp (+37-2)
- (added) clang/test/CodeGen/scoped-atomic-ops.c (+331)
- (modified) clang/test/Preprocessor/init-aarch64.c (+5)
- (modified) clang/test/Preprocessor/init-loongarch.c (+10)
- (modified) clang/test/Preprocessor/init.c (+20)
- (added) clang/test/Sema/scoped-atomic-ops.c (+101)
``````````diff
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index a9c4c67a60e8e8e..a41f2d66b37b69d 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -6498,7 +6498,7 @@ class AtomicExpr : public Expr {
return cast<Expr>(SubExprs[ORDER_FAIL]);
}
Expr *getVal2() const {
- if (Op == AO__atomic_exchange)
+ if (Op == AO__atomic_exchange || Op == AO__scoped_atomic_exchange)
return cast<Expr>(SubExprs[ORDER_FAIL]);
assert(NumSubExprs > VAL2);
return cast<Expr>(SubExprs[VAL2]);
@@ -6539,7 +6539,9 @@ class AtomicExpr : public Expr {
getOp() == AO__opencl_atomic_compare_exchange_weak ||
getOp() == AO__hip_atomic_compare_exchange_weak ||
getOp() == AO__atomic_compare_exchange ||
- getOp() == AO__atomic_compare_exchange_n;
+ getOp() == AO__atomic_compare_exchange_n ||
+ getOp() == AO__scoped_atomic_compare_exchange ||
+ getOp() == AO__scoped_atomic_compare_exchange_n;
}
bool isOpenCL() const {
@@ -6569,13 +6571,13 @@ class AtomicExpr : public Expr {
/// \return empty atomic scope model if the atomic op code does not have
/// scope operand.
static std::unique_ptr<AtomicScopeModel> getScopeModel(AtomicOp Op) {
- auto Kind =
- (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max)
- ? AtomicScopeModelKind::OpenCL
- : (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max)
- ? AtomicScopeModelKind::HIP
- : AtomicScopeModelKind::None;
- return AtomicScopeModel::create(Kind);
+ if (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max)
+ return AtomicScopeModel::create(AtomicScopeModelKind::OpenCL);
+ else if (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max)
+ return AtomicScopeModel::create(AtomicScopeModelKind::HIP);
+ else if (Op >= AO__scoped_atomic_load && Op <= AO__scoped_atomic_fetch_max)
+ return AtomicScopeModel::create(AtomicScopeModelKind::Generic);
+ return AtomicScopeModel::create(AtomicScopeModelKind::None);
}
/// Get atomic scope model.
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index ec39e926889b936..4dcbaf8a7beaa65 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -904,6 +904,32 @@ BUILTIN(__atomic_signal_fence, "vi", "n")
BUILTIN(__atomic_always_lock_free, "bzvCD*", "nE")
BUILTIN(__atomic_is_lock_free, "bzvCD*", "nE")
+// GNU atomic builtins with atomic scopes.
+ATOMIC_BUILTIN(__scoped_atomic_load, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_load_n, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_store, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_store_n, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_exchange, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_exchange_n, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_compare_exchange, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_compare_exchange_n, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_add, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_sub, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_and, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_or, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_xor, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_nand, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_add_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_sub_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_and_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_or_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_xor_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_max_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_min_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_nand_fetch, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_min, "v.", "t")
+ATOMIC_BUILTIN(__scoped_atomic_fetch_max, "v.", "t")
+
// OpenCL 2.0 atomic builtins.
ATOMIC_BUILTIN(__opencl_atomic_init, "v.", "t")
ATOMIC_BUILTIN(__opencl_atomic_load, "v.", "t")
diff --git a/clang/include/clang/Basic/SyncScope.h b/clang/include/clang/Basic/SyncScope.h
index 7919f64c6daf97b..bc7ec7b5cf777ef 100644
--- a/clang/include/clang/Basic/SyncScope.h
+++ b/clang/include/clang/Basic/SyncScope.h
@@ -40,6 +40,11 @@ namespace clang {
/// Update getAsString.
///
enum class SyncScope {
+ SystemScope,
+ DeviceScope,
+ WorkgroupScope,
+ WavefrontScope,
+ SingleScope,
HIPSingleThread,
HIPWavefront,
HIPWorkgroup,
@@ -54,6 +59,16 @@ enum class SyncScope {
inline llvm::StringRef getAsString(SyncScope S) {
switch (S) {
+ case SyncScope::SystemScope:
+ return "system_scope";
+ case SyncScope::DeviceScope:
+ return "device_scope";
+ case SyncScope::WorkgroupScope:
+ return "workgroup_scope";
+ case SyncScope::WavefrontScope:
+ return "wavefront_scope";
+ case SyncScope::SingleScope:
+ return "single_scope";
case SyncScope::HIPSingleThread:
return "hip_singlethread";
case SyncScope::HIPWavefront:
@@ -77,7 +92,7 @@ inline llvm::StringRef getAsString(SyncScope S) {
}
/// Defines the kind of atomic scope models.
-enum class AtomicScopeModelKind { None, OpenCL, HIP };
+enum class AtomicScopeModelKind { None, OpenCL, HIP, Generic };
/// Defines the interface for synch scope model.
class AtomicScopeModel {
@@ -205,6 +220,56 @@ class AtomicScopeHIPModel : public AtomicScopeModel {
}
};
+/// Defines the generic atomic scope model.
+class AtomicScopeGenericModel : public AtomicScopeModel {
+public:
+ /// The enum values match predefined built-in macros __ATOMIC_SCOPE_*.
+ enum ID {
+ System = 0,
+ Device = 1,
+ Workgroup = 2,
+ Wavefront = 3,
+ Single = 4,
+ Last = Single
+ };
+
+ AtomicScopeGenericModel() = default;
+
+ SyncScope map(unsigned S) const override {
+ switch (static_cast<ID>(S)) {
+ case Device:
+ return SyncScope::DeviceScope;
+ case System:
+ return SyncScope::SystemScope;
+ case Workgroup:
+ return SyncScope::WorkgroupScope;
+ case Wavefront:
+ return SyncScope::WavefrontScope;
+ case Single:
+ return SyncScope::SingleScope;
+ }
+ llvm_unreachable("Invalid language sync scope value");
+ }
+
+ bool isValid(unsigned S) const override {
+ return S >= static_cast<unsigned>(System) &&
+ S <= static_cast<unsigned>(Last);
+ }
+
+ ArrayRef<unsigned> getRuntimeValues() const override {
+ static_assert(Last == Single, "Does not include all sync scopes");
+ static const unsigned Scopes[] = {
+ static_cast<unsigned>(Device), static_cast<unsigned>(System),
+ static_cast<unsigned>(Workgroup), static_cast<unsigned>(Wavefront),
+ static_cast<unsigned>(Single)};
+ return llvm::ArrayRef(Scopes);
+ }
+
+ unsigned getFallBackValue() const override {
+ return static_cast<unsigned>(System);
+ }
+};
+
inline std::unique_ptr<AtomicScopeModel>
AtomicScopeModel::create(AtomicScopeModelKind K) {
switch (K) {
@@ -214,6 +279,8 @@ AtomicScopeModel::create(AtomicScopeModelKind K) {
return std::make_unique<AtomicScopeOpenCLModel>();
case AtomicScopeModelKind::HIP:
return std::make_unique<AtomicScopeHIPModel>();
+ case AtomicScopeModelKind::Generic:
+ return std::make_unique<AtomicScopeGenericModel>();
}
llvm_unreachable("Invalid atomic scope model kind");
}
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 55c6b732b7081f4..b125fc676da8419 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -4887,6 +4887,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) {
case AO__atomic_load_n:
return 2;
+ case AO__scoped_atomic_load_n:
case AO__opencl_atomic_load:
case AO__hip_atomic_load:
case AO__c11_atomic_store:
@@ -4921,6 +4922,26 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) {
case AO__atomic_fetch_max:
return 3;
+ case AO__scoped_atomic_load:
+ case AO__scoped_atomic_store:
+ case AO__scoped_atomic_store_n:
+ case AO__scoped_atomic_fetch_add:
+ case AO__scoped_atomic_fetch_sub:
+ case AO__scoped_atomic_fetch_and:
+ case AO__scoped_atomic_fetch_or:
+ case AO__scoped_atomic_fetch_xor:
+ case AO__scoped_atomic_fetch_nand:
+ case AO__scoped_atomic_add_fetch:
+ case AO__scoped_atomic_sub_fetch:
+ case AO__scoped_atomic_and_fetch:
+ case AO__scoped_atomic_or_fetch:
+ case AO__scoped_atomic_xor_fetch:
+ case AO__scoped_atomic_nand_fetch:
+ case AO__scoped_atomic_min_fetch:
+ case AO__scoped_atomic_max_fetch:
+ case AO__scoped_atomic_fetch_min:
+ case AO__scoped_atomic_fetch_max:
+ case AO__scoped_atomic_exchange_n:
case AO__hip_atomic_exchange:
case AO__hip_atomic_fetch_add:
case AO__hip_atomic_fetch_sub:
@@ -4942,6 +4963,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) {
case AO__atomic_exchange:
return 4;
+ case AO__scoped_atomic_exchange:
case AO__c11_atomic_compare_exchange_strong:
case AO__c11_atomic_compare_exchange_weak:
return 5;
@@ -4952,6 +4974,10 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) {
case AO__atomic_compare_exchange:
case AO__atomic_compare_exchange_n:
return 6;
+
+ case AO__scoped_atomic_compare_exchange:
+ case AO__scoped_atomic_compare_exchange_n:
+ return 7;
}
llvm_unreachable("unknown atomic op");
}
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index ab4a013de5f552c..c04cb313c3387a3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -1841,6 +1841,7 @@ void StmtPrinter::VisitAtomicExpr(AtomicExpr *Node) {
PrintExpr(Node->getPtr());
if (Node->getOp() != AtomicExpr::AO__c11_atomic_load &&
Node->getOp() != AtomicExpr::AO__atomic_load_n &&
+ Node->getOp() != AtomicExpr::AO__scoped_atomic_load_n &&
Node->getOp() != AtomicExpr::AO__opencl_atomic_load &&
Node->getOp() != AtomicExpr::AO__hip_atomic_load) {
OS << ", ";
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index f7c597e181b0bd9..b7636522f2c8932 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -508,9 +508,11 @@ static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
default:
llvm_unreachable("Unexpected min/max operation");
case AtomicExpr::AO__atomic_max_fetch:
+ case AtomicExpr::AO__scoped_atomic_max_fetch:
Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
break;
case AtomicExpr::AO__atomic_min_fetch:
+ case AtomicExpr::AO__scoped_atomic_min_fetch:
Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
break;
}
@@ -545,7 +547,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
FailureOrder, Size, Order, Scope);
return;
case AtomicExpr::AO__atomic_compare_exchange:
- case AtomicExpr::AO__atomic_compare_exchange_n: {
+ case AtomicExpr::AO__atomic_compare_exchange_n:
+ case AtomicExpr::AO__scoped_atomic_compare_exchange:
+ case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
Val1, Val2, FailureOrder, Size, Order, Scope);
@@ -578,7 +582,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
case AtomicExpr::AO__opencl_atomic_load:
case AtomicExpr::AO__hip_atomic_load:
case AtomicExpr::AO__atomic_load_n:
- case AtomicExpr::AO__atomic_load: {
+ case AtomicExpr::AO__atomic_load:
+ case AtomicExpr::AO__scoped_atomic_load_n:
+ case AtomicExpr::AO__scoped_atomic_load: {
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
Load->setAtomic(Order, Scope);
Load->setVolatile(E->isVolatile());
@@ -590,7 +596,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
case AtomicExpr::AO__opencl_atomic_store:
case AtomicExpr::AO__hip_atomic_store:
case AtomicExpr::AO__atomic_store:
- case AtomicExpr::AO__atomic_store_n: {
+ case AtomicExpr::AO__atomic_store_n:
+ case AtomicExpr::AO__scoped_atomic_store:
+ case AtomicExpr::AO__scoped_atomic_store_n: {
llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
Store->setAtomic(Order, Scope);
@@ -603,10 +611,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
case AtomicExpr::AO__opencl_atomic_exchange:
case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__atomic_exchange:
+ case AtomicExpr::AO__scoped_atomic_exchange_n:
+ case AtomicExpr::AO__scoped_atomic_exchange:
Op = llvm::AtomicRMWInst::Xchg;
break;
case AtomicExpr::AO__atomic_add_fetch:
+ case AtomicExpr::AO__scoped_atomic_add_fetch:
PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
: llvm::Instruction::Add;
[[fallthrough]];
@@ -614,11 +625,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
case AtomicExpr::AO__hip_atomic_fetch_add:
case AtomicExpr::AO__opencl_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
+ case AtomicExpr::AO__scoped_atomic_fetch_add:
Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
: llvm::AtomicRMWInst::Add;
break;
case AtomicExpr::AO__atomic_sub_fetch:
+ case AtomicExpr::AO__scoped_atomic_sub_fetch:
PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
: llvm::Instruction::Sub;
[[fallthrough]];
@@ -626,17 +639,20 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
case AtomicExpr::AO__hip_atomic_fetch_sub:
case AtomicExpr::AO__opencl_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
+ case AtomicExpr::AO__scoped_atomic_fetch_sub:
Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
: llvm::AtomicRMWInst::Sub;
break;
case AtomicExpr::AO__atomic_min_fetch:
+ case AtomicExpr::AO__scoped_atomic_min_fetch:
PostOpMinMax = true;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_min:
case AtomicExpr::AO__hip_atomic_fetch_min:
case AtomicExpr::AO__opencl_atomic_fetch_min:
case AtomicExpr::AO__atomic_fetch_min:
+ case AtomicExpr::AO__scoped_atomic_fetch_min:
Op = E->getValueType()->isFloatingType()
? llvm::AtomicRMWInst::FMin
: (E->getValueType()->isSignedIntegerType()
@@ -645,12 +661,14 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
break;
case AtomicExpr::AO__atomic_max_fetch:
+ case AtomicExpr::AO__scoped_atomic_max_fetch:
PostOpMinMax = true;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_max:
case AtomicExpr::AO__hip_atomic_fetch_max:
case AtomicExpr::AO__opencl_atomic_fetch_max:
case AtomicExpr::AO__atomic_fetch_max:
+ case AtomicExpr::AO__scoped_atomic_fetch_max:
Op = E->getValueType()->isFloatingType()
? llvm::AtomicRMWInst::FMax
: (E->getValueType()->isSignedIntegerType()
@@ -659,40 +677,48 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
break;
case AtomicExpr::AO__atomic_and_fetch:
+ case AtomicExpr::AO__scoped_atomic_and_fetch:
PostOp = llvm::Instruction::And;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__hip_atomic_fetch_and:
case AtomicExpr::AO__opencl_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
+ case AtomicExpr::AO__scoped_atomic_fetch_and:
Op = llvm::AtomicRMWInst::And;
break;
case AtomicExpr::AO__atomic_or_fetch:
+ case AtomicExpr::AO__scoped_atomic_or_fetch:
PostOp = llvm::Instruction::Or;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__hip_atomic_fetch_or:
case AtomicExpr::AO__opencl_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
+ case AtomicExpr::AO__scoped_atomic_fetch_or:
Op = llvm::AtomicRMWInst::Or;
break;
case AtomicExpr::AO__atomic_xor_fetch:
+ case AtomicExpr::AO__scoped_atomic_xor_fetch:
PostOp = llvm::Instruction::Xor;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__hip_atomic_fetch_xor:
case AtomicExpr::AO__opencl_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
+ case AtomicExpr::AO__scoped_atomic_fetch_xor:
Op = llvm::AtomicRMWInst::Xor;
break;
case AtomicExpr::AO__atomic_nand_fetch:
+ case AtomicExpr::AO__scoped_atomic_nand_fetch:
PostOp = llvm::Instruction::And; // the NOT is special cased below
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_nand:
case AtomicExpr::AO__atomic_fetch_nand:
+ case AtomicExpr::AO__scoped_atomic_fetch_nand:
Op = llvm::AtomicRMWInst::Nand;
break;
}
@@ -712,7 +738,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
else if (PostOp)
Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
LoadVal1);
- if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
+ if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
+ E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
Result = CGF.Builder.CreateNot(Result);
CGF.Builder.CreateStore(Result, Dest);
}
@@ -865,17 +892,21 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
case AtomicExpr::AO__opencl_atomic_load:
case AtomicExpr::AO__hip_atomic_load:
case AtomicExpr::AO__atomic_load_n:
+ case AtomicExpr::AO__scoped_atomic_load_n:
break;
case AtomicExpr::AO__atomic_load:
+ case AtomicExpr::AO__scoped_atomic_load:
Dest = EmitPointerWithAlignment(E->getVal1());
break;
case AtomicExpr::AO__atomic_store:
+ case AtomicExpr::AO__scoped_atomic_store:
Val1 = EmitPointerWithAlignment(E->getVal1());
break;
case AtomicExpr::AO__atomic_exchange:
+ case AtomicExpr::AO__scoped_atomic_exchange:
Val1 = EmitPointerWithAlignment(E->getVal1());
Dest = EmitPointerWithAlignment(E->getVal2());
break;
@@ -888,14 +919,19 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
case AtomicExpr::AO__atomic_compare_exchange_n:
case AtomicExpr::AO__atomic_compare_exchange:
+ case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
+ case AtomicExpr::AO__scoped_atomic_compare_exchange:
Val1 = EmitPointerWithAlignment(E->getVal1());
- if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
+ if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
+ E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
Val2 = EmitPointerWithAlignment(E->getVal2());
else
Val2 = EmitValToTemp(*this, E->getVal2());
OrderFail = EmitScalarExpr(E->getOrderFail());
if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
- E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
+ E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
+ E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
+ E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
IsWeak = EmitScalarExpr(E->getWeak());
break;
@@ -929,6 +965,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
case AtomicExpr::AO__atomic_max_fetch:
case AtomicExpr::AO__atomic_min_fetch:
case AtomicExpr::AO__atomic_sub_fetch:
+ case AtomicExpr::AO__scoped_atomic_fetch_add:
+ case AtomicExpr::AO__scoped_atomic_fetch_max:
+ case AtomicExpr::AO__scoped_atomic_fetch_min:
+ case AtomicExpr::AO__scoped_atomic_fetch_sub:
+ case AtomicExpr::AO__scoped_atomic_add_fetch:
+ case AtomicExpr::AO__scoped_atomic_max_fetch:
+ case AtomicExpr::AO__scoped_atomic_min_fetch:
+ case AtomicExpr::AO__scoped_atomic_sub_fetch:
case AtomicExpr::AO__c11_atomic_fetch_max:
case AtomicExpr::AO__c11_atomic_fetch_min:
case AtomicExpr::AO__opencl_atomic_fetch_max:
@@ -946,6 +990,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
case AtomicExpr::AO__hip_atomic_exchange:
case AtomicExpr::AO__atomic_...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/72280
More information about the cfe-commits
mailing list