[clang] [CIR] Implement 'simple' atomic inc/dec. (PR #222730)
via cfe-commits
cfe-commits at lists.llvm.org
Thu Sep 10 11:08:21 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang
Author: Erich Keane (erichkeane)
<details>
<summary>Changes</summary>
This only works for types where a compare-exchange instruction (or just a lock-free implementation matters) is possible, and a loop isn't necessary. This is basically just a conversion of classic-codegen,
except it clarifies some of the logic (particularly around bools).
---
Full diff: https://github.com/llvm/llvm-project/pull/222730.diff
2 Files Affected:
- (modified) clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp (+81-3)
- (modified) clang/test/CIR/CodeGen/atomic.c (+215)
``````````diff
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index fb833241c1f81..3a5e928c51c45 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -628,10 +628,88 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
mlir::Value value;
mlir::Value input;
- if (type->getAs<AtomicType>()) {
+ if (const AtomicType *atomicTy = type->getAs<AtomicType>()) {
+ QualType valType = atomicTy->getValueType();
+ mlir::Location loc = cgf.getLoc(e->getSourceRange());
+ bool isInc = e->isIncrementOp();
+ bool isPre = e->isPrefix();
+
+ // Bools are always set-to-true, as decrement isn't legal on bools.
+ if (valType->isBooleanType()) {
+ assert(isInc);
+ // Atomic operations require an integer type; reinterpret the bool
+ // pointer as a pointer to its underlying storage integer type.
+ cir::IntType intTy = builder.getUInt8Ty();
+ mlir::Value one = builder.getConstInt(loc, intTy, 1);
+ Address intAddr = lv.getAddress().withElementType(builder, intTy);
+
+ if (isPre) {
+ // Pre-increment: atomically store true, return true.
+ cir::StoreOp store = builder.createStore(loc, one, intAddr);
+ store.setMemOrder(cir::MemOrder::SequentiallyConsistent);
+ if (lv.isVolatileQualified())
+ store.setIsVolatile(true);
+ return builder.getTrue(loc);
+ }
+ // Post-increment: atomically exchange with true, return old value.
+ auto xchg = cir::AtomicXchgOp::create(
+ builder, loc, intAddr.getPointer(), one,
+ cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System,
+ lv.isVolatileQualified());
+ return builder.createCast(cir::CastKind::int_to_bool, xchg.getResult(),
+ builder.getBoolTy());
+ }
+
+ // Special case for atomic increment / decrement on integers, emit
+ // atomicrmw instructions. We skip this if we want to be doing overflow
+ // checking, and fall into the slow path with the atomic cmpxchg loop.
+ if (!valType->isBooleanType() && valType->isIntegerType() &&
+ !(valType->isUnsignedIntegerType() &&
+ cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) &&
+ cgf.getLangOpts().getSignedOverflowBehavior() !=
+ LangOptions::SOB_Trapping) {
+ mlir::Type intTy = cgf.convertType(valType);
+ mlir::Value one = builder.getConstInt(loc, intTy, 1);
+ cir::AtomicFetchKind kind =
+ isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub;
+ auto rmw = cir::AtomicFetchOp::create(
+ builder, loc, lv.getPointer(), one, kind,
+ cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System,
+ lv.isVolatileQualified(), /*fetch_first=*/true);
+ mlir::Value oldVal = rmw->getResult(0);
+ // Prefix returns new value; postfix returns old value.
+ return isPre ? emitIncOrDec(e, oldVal) : oldVal;
+ }
+
+ // Special case for atomic increment/decrement on floats.
+ // Bail out non-power-of-2-sized floating point types (e.g., x86_fp80).
+ if (valType->isFloatingType()) {
+ CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
+ mlir::Type fpTy = cgf.convertType(valType);
+ auto fpType = mlir::cast<cir::FPTypeInterface>(fpTy);
+ // Bail on non-power-of-2 types (e.g., x86_fp80 is 80 bits).
+ if (llvm::has_single_bit(fpType.getWidth())) {
+ mlir::Value amt = builder.getConstFP(
+ loc, fpTy, llvm::APFloat(fpType.getFloatSemantics(), 1));
+ cir::AtomicFetchKind kind =
+ isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub;
+ auto rmw = cir::AtomicFetchOp::create(
+ builder, loc, lv.getPointer(), amt, kind,
+ cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System,
+ lv.isVolatileQualified(), /*fetch_first=*/true);
+ mlir::Value oldVal = rmw->getResult(0);
+ if (isPre)
+ return isInc ? builder.createFAdd(loc, oldVal, amt)
+ : builder.createFSub(loc, oldVal, amt);
+ return oldVal;
+ } else {
+ cgf.cgm.errorNYI(e->getSourceRange(),
+ "Atomic inc/dec of non-power-of-2 float");
+ }
+ }
+
+ // Otherwise we need a loop on compare-exchange.
cgf.cgm.errorNYI(e->getSourceRange(), "Atomic inc/dec");
- // TODO(cir): This is not correct, but it will produce reasonable code
- // until atomic operations are implemented.
value = cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue();
input = value;
} else {
diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c
index 21b99331fb417..f3913ea28072e 100644
--- a/clang/test/CIR/CodeGen/atomic.c
+++ b/clang/test/CIR/CodeGen/atomic.c
@@ -4022,3 +4022,218 @@ void store_atomic_different_size(S a) {
// OGCG: %[[ATOMIC_TMP:.*]] = load i32, ptr %[[ATOMIC_TMP_ADDR]], align 4
// OGCG: store atomic i32 %[[ATOMIC_TMP]], ptr %[[B_ADDR]] seq_cst, align 4
}
+
+void atomic_pre_inc(_Atomic(int) *p) { ++(*p); }
+
+// CIR-LABEL: @atomic_pre_inc
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i
+// CIR: cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i
+
+// LLVM-LABEL: @atomic_pre_inc
+// LLVM: atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4
+
+// OGCG-LABEL: @atomic_pre_inc
+// OGCG: atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4
+
+int atomic_post_inc(_Atomic(int) *p) { return (*p)++; }
+
+// CIR-LABEL: @atomic_post_inc
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i
+// CIR: cir.store %[[OLD]], %{{.+}} : !s32i, !cir.ptr<!s32i>
+
+// LLVM-LABEL: @atomic_post_inc
+// LLVM: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4
+// LLVM: store i32 %[[OLD]], ptr %{{.+}}, align 4
+// LLVM: %[[RET:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM: ret i32 %[[RET]]
+
+// OGCG-LABEL: @atomic_post_inc
+// OGCG: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4
+// OGCG: ret i32 %[[OLD]]
+
+void atomic_pre_dec(_Atomic(int) *p) { --(*p); }
+
+// CIR-LABEL: @atomic_pre_dec
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i
+// CIR: cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i
+
+// LLVM-LABEL: @atomic_pre_dec
+// LLVM: atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4
+
+// OGCG-LABEL: @atomic_pre_dec
+// OGCG: atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4
+
+int atomic_post_dec(_Atomic(int) *p) { return (*p)--; }
+
+// CIR-LABEL: @atomic_post_dec
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[OLD:.+]] = cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i
+// CIR: cir.store %[[OLD]], %{{.+}} : !s32i, !cir.ptr<!s32i>
+
+// LLVM-LABEL: @atomic_post_dec
+// LLVM: %[[OLD:.+]] = atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4
+// LLVM: store i32 %[[OLD]], ptr %{{.+}}, align 4
+// LLVM: %[[RET:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM: ret i32 %[[RET]]
+
+// OGCG-LABEL: @atomic_post_dec
+// OGCG: %[[OLD:.+]] = atomicrmw sub ptr %{{.+}}, i32 1 seq_cst, align 4
+// OGCG: ret i32 %[[OLD]]
+
+int atomic_pre_inc_used(_Atomic(int) *p) { return ++(*p); }
+
+// CIR-LABEL: @atomic_pre_inc_used
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[ONE]] : (!cir.ptr<!s32i>, !s32i) -> !s32i
+// CIR: %[[NEW:.+]] = cir.inc %[[OLD]] : !s32i
+// CIR: cir.store %[[NEW]], %{{.+}} : !s32i, !cir.ptr<!s32i>
+
+// LLVM-LABEL: @atomic_pre_inc_used
+// LLVM: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4
+// LLVM: %[[NEW:.+]] = add i32 %[[OLD]], 1
+// LLVM: store i32 %[[NEW]], ptr %{{.+}}, align 4
+// LLVM: %[[RET:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM: ret i32 %[[RET]]
+
+// OGCG-LABEL: @atomic_pre_inc_used
+// OGCG: %[[OLD:.+]] = atomicrmw add ptr %{{.+}}, i32 1 seq_cst, align 4
+// OGCG: %[[NEW:.+]] = add i32 %[[OLD]], 1
+// OGCG: ret i32 %[[NEW]]
+
+// Tests for atomic increment on _Bool.
+
+void atomic_bool_pre_inc_void(_Atomic _Bool *p) { ++(*p); }
+
+// CIR-LABEL: @atomic_bool_pre_inc_void
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.bool>>, !cir.ptr<!cir.bool>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !u8i
+// CIR: %[[IPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!cir.bool> -> !cir.ptr<!u8i>
+// CIR: cir.store {{.*}} atomic(seq_cst) %[[ONE]], %[[IPTR]] : !u8i, !cir.ptr<!u8i>
+
+// LLVM-LABEL: @atomic_bool_pre_inc_void
+// LLVM: store atomic i8 1, ptr %{{.+}} seq_cst, align 1
+
+// OGCG-LABEL: @atomic_bool_pre_inc_void
+// OGCG: store atomic i8 1, ptr %{{.+}} seq_cst, align 1
+
+_Bool atomic_bool_pre_inc(_Atomic _Bool *p) { return ++(*p); }
+
+// CIR-LABEL: @atomic_bool_pre_inc
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.bool>>, !cir.ptr<!cir.bool>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !u8i
+// CIR: %[[IPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!cir.bool> -> !cir.ptr<!u8i>
+// CIR: cir.store {{.*}} atomic(seq_cst) %[[ONE]], %[[IPTR]] : !u8i, !cir.ptr<!u8i>
+// CIR: cir.const #true
+
+// LLVM-LABEL: @atomic_bool_pre_inc
+// LLVM: store atomic i8 1, ptr %{{.+}} seq_cst, align 1
+
+// OGCG-LABEL: @atomic_bool_pre_inc
+// OGCG: store atomic i8 1, ptr %{{.+}} seq_cst, align 1
+// OGCG: ret i1 true
+
+_Bool atomic_bool_post_inc(_Atomic _Bool *p) { return (*p)++; }
+
+// CIR-LABEL: @atomic_bool_post_inc
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.bool>>, !cir.ptr<!cir.bool>
+// CIR: %[[ONE:.+]] = cir.const #cir.int<1> : !u8i
+// CIR: %[[IPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!cir.bool> -> !cir.ptr<!u8i>
+// CIR: %[[OLD:.+]] = cir.atomic.xchg seq_cst syncscope(system) %[[IPTR]], %[[ONE]] : (!cir.ptr<!u8i>, !u8i) -> !u8i
+// CIR: cir.cast int_to_bool %[[OLD]] : !u8i -> !cir.bool
+
+// LLVM-LABEL: @atomic_bool_post_inc
+// LLVM: %[[OLD:.+]] = atomicrmw xchg ptr %{{.+}}, i8 1 seq_cst, align 1
+
+// OGCG-LABEL: @atomic_bool_post_inc
+// OGCG: %[[OLD:.+]] = atomicrmw xchg ptr %{{.+}}, i8 1 seq_cst, align 1
+
+// Tests for atomic increment/decrement on float.
+
+void atomic_float_pre_inc(_Atomic(float) *p) { ++(*p); }
+
+// CIR-LABEL: @atomic_float_pre_inc
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float>
+// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float
+// CIR: cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float
+
+// LLVM-LABEL: @atomic_float_pre_inc
+// LLVM: atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+
+// OGCG-LABEL: @atomic_float_pre_inc
+// OGCG: atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+
+float atomic_float_post_inc(_Atomic(float) *p) { return (*p)++; }
+
+// CIR-LABEL: @atomic_float_post_inc
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float>
+// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float
+// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float
+// CIR: cir.store %[[OLD]], %{{.+}} : !cir.float, !cir.ptr<!cir.float>
+
+// LLVM-LABEL: @atomic_float_post_inc
+// LLVM: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+// LLVM: store float %[[OLD]], ptr %{{.+}}, align 4
+// LLVM: %[[RET:.+]] = load float, ptr %{{.+}}, align 4
+// LLVM: ret float %[[RET]]
+
+// OGCG-LABEL: @atomic_float_post_inc
+// OGCG: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+// OGCG: ret float %[[OLD]]
+
+float atomic_float_pre_inc_used(_Atomic(float) *p) { return ++(*p); }
+
+// CIR-LABEL: @atomic_float_pre_inc_used
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float>
+// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float
+// CIR: %[[OLD:.+]] = cir.atomic.fetch add seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float
+// CIR: %[[NEW:.+]] = cir.fadd %[[OLD]], %[[AMT]] : !cir.float
+// CIR: cir.store %[[NEW]], %{{.+}} : !cir.float, !cir.ptr<!cir.float>
+
+// LLVM-LABEL: @atomic_float_pre_inc_used
+// LLVM: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+// LLVM: %[[NEW:.+]] = fadd float %[[OLD]], 1.000000e+00
+// LLVM: store float %[[NEW]], ptr %{{.+}}, align 4
+// LLVM: %[[RET:.+]] = load float, ptr %{{.+}}, align 4
+// LLVM: ret float %[[RET]]
+
+// OGCG-LABEL: @atomic_float_pre_inc_used
+// OGCG: %[[OLD:.+]] = atomicrmw fadd ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+// OGCG: %[[NEW:.+]] = fadd float %[[OLD]], 1.000000e+00
+// OGCG: ret float %[[NEW]]
+
+void atomic_float_pre_dec(_Atomic(float) *p) { --(*p); }
+
+// CIR-LABEL: @atomic_float_pre_dec
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float>
+// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float
+// CIR: cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float
+
+// LLVM-LABEL: @atomic_float_pre_dec
+// LLVM: atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+
+// OGCG-LABEL: @atomic_float_pre_dec
+// OGCG: atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+
+float atomic_float_post_dec(_Atomic(float) *p) { return (*p)--; }
+
+// CIR-LABEL: @atomic_float_post_dec
+// CIR: %[[PTR:.+]] = cir.load deref {{.*}} : !cir.ptr<!cir.ptr<!cir.float>>, !cir.ptr<!cir.float>
+// CIR: %[[AMT:.+]] = cir.const #cir.fp<1.{{.*}}> : !cir.float
+// CIR: %[[OLD:.+]] = cir.atomic.fetch sub seq_cst syncscope(system) fetch_first %[[PTR]], %[[AMT]] : (!cir.ptr<!cir.float>, !cir.float) -> !cir.float
+// CIR: cir.store %[[OLD]], %{{.+}} : !cir.float, !cir.ptr<!cir.float>
+
+// LLVM-LABEL: @atomic_float_post_dec
+// LLVM: %[[OLD:.+]] = atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+// LLVM: store float %[[OLD]], ptr %{{.+}}, align 4
+// LLVM: %[[RET:.+]] = load float, ptr %{{.+}}, align 4
+// LLVM: ret float %[[RET]]
+
+// OGCG-LABEL: @atomic_float_post_dec
+// OGCG: %[[OLD:.+]] = atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
+// OGCG: ret float %[[OLD]]
``````````
</details>
https://github.com/llvm/llvm-project/pull/222730
More information about the cfe-commits
mailing list