[clang] [CIR] Correct 2 lowering bugs of atomic cmp-xchng builtins (PR #227054)
Erich Keane via cfe-commits
cfe-commits at lists.llvm.org
Mon Sep 28 10:58:41 PDT 2026
https://github.com/erichkeane updated https://github.com/llvm/llvm-project/pull/227054
>From 3818998712fa877add45171ab103e6c725795ac6 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane at nvidia.com>
Date: Mon, 28 Sep 2026 10:21:17 -0700
Subject: [PATCH 1/3] [CIR] Correct 2 lowering bugs of atomic cmp-xchng
builtins
This patch fixese two bugs that showed up in a benchmark.
First; convertToAtomicIntPointer was zero-filling the source object
directly, rather than the temporary. The result was that anything that
would not be overwritten thanks to the power-of-2 write, would be
incorrect, and corrupted.
Second; emitAtomicCmpXchg didn't set the 'old' value back into the real
object. This ends up doing an additional argument on this function that
better matches classic-codegen.
---
clang/lib/CIR/CodeGen/CIRGenAtomic.cpp | 129 ++++++++++++++-----------
clang/test/CIR/CodeGen/atomic.c | 72 ++++++++++++--
2 files changed, 141 insertions(+), 60 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
index e89be4fc60975..fc47e732bc8a0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
@@ -212,9 +212,9 @@ Address AtomicInfo::convertToAtomicIntPointer(Address addr,
cgf.getContext().toCharUnitsFromBits(atomicSizeInBits).getQuantity();
mlir::Value memSetSize = builder.getConstInt(loc, cgf.cgm.uInt64Ty, size);
addr = addr.withElementType(builder, cgf.cgm.voidTy);
- builder.createMemSet(loc, addr, zero, memSetSize);
-
tmp = tmp.withElementType(builder, cgf.cgm.voidTy);
+ builder.createMemSet(loc, tmp, zero, memSetSize);
+
builder.createMemCpy(
loc, tmp, addr,
builder.getConstInt(loc, cgf.cgm.uInt64Ty,
@@ -484,8 +484,8 @@ static void emitMemOrderCaseLabel(CIRGenBuilderTy &builder, mlir::Location loc,
static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,
Address dest, Address ptr, Address val1,
- Address val2, uint64_t size,
- cir::MemOrder successOrder,
+ Address val2, Address expectedResult,
+ uint64_t size, cir::MemOrder successOrder,
cir::MemOrder failureOrder,
cir::SyncScopeKind scope) {
mlir::Location loc = cgf.getLoc(e->getSourceRange());
@@ -506,17 +506,30 @@ static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,
cmpxchg.setWeak(isWeak);
mlir::Value failed = builder.createNot(cmpxchg.getSuccess());
- cir::IfOp::create(builder, loc, failed, /*withElseRegion=*/false,
- [&](mlir::OpBuilder &, mlir::Location) {
- auto ptrTy = mlir::cast<cir::PointerType>(
- val1.getPointer().getType());
- if (val1.getElementType() != ptrTy.getPointee()) {
- val1 = val1.withPointer(builder.createPtrBitcast(
- val1.getPointer(), val1.getElementType()));
- }
- builder.createStore(loc, cmpxchg.getOld(), val1);
- builder.createYield(loc);
- });
+ cir::IfOp::create(
+ builder, loc, failed, /*withElseRegion=*/false,
+ [&](mlir::OpBuilder &, mlir::Location) {
+ uint64_t expectedSizeInBytes = cgf.cgm.getDataLayout().getTypeStoreSize(
+ expectedResult.getElementType());
+
+ if (expectedSizeInBytes == size) {
+ Address storeAddr = expectedResult.withElementType(
+ builder, cmpxchg.getOld().getType());
+ builder.createStore(loc, cmpxchg.getOld(), storeAddr);
+ } else {
+ Address oldTmp = cgf.createTempAlloca(
+ cmpxchg.getOld().getType(), ptr.getAlignment(), loc, "old.tmp");
+ builder.createStore(loc, cmpxchg.getOld(), oldTmp);
+
+ Address oldTmpVoid = oldTmp.withElementType(builder, cgf.cgm.voidTy);
+ Address expectedVoid =
+ expectedResult.withElementType(builder, cgf.cgm.voidTy);
+ builder.createMemCpy(
+ loc, expectedVoid, oldTmpVoid,
+ builder.getConstInt(loc, cgf.cgm.uInt64Ty, expectedSizeInBytes));
+ }
+ builder.createYield(loc);
+ });
// Update the memory at Dest with Success's value.
cgf.emitStoreOfScalar(cmpxchg.getSuccess(),
@@ -524,12 +537,10 @@ static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,
/*isInit=*/false);
}
-static void emitAtomicCmpXchgFailureSet(CIRGenFunction &cgf, AtomicExpr *e,
- bool isWeak, Address dest, Address ptr,
- Address val1, Address val2,
- Expr *failureOrderExpr, uint64_t size,
- cir::MemOrder successOrder,
- cir::SyncScopeKind scope) {
+static void emitAtomicCmpXchgFailureSet(
+ CIRGenFunction &cgf, AtomicExpr *e, bool isWeak, Address dest, Address ptr,
+ Address val1, Address val2, Address expectedResult, Expr *failureOrderExpr,
+ uint64_t size, cir::MemOrder successOrder, cir::SyncScopeKind scope) {
Expr::EvalResult failureOrderEval;
if (failureOrderExpr->EvaluateAsInt(failureOrderEval, cgf.getContext())) {
uint64_t failureOrderInt = failureOrderEval.Val.getInt().getZExtValue();
@@ -560,8 +571,8 @@ static void emitAtomicCmpXchgFailureSet(CIRGenFunction &cgf, AtomicExpr *e,
// success argument". This condition has been lifted and the only
// precondition is 31.7.2.18. Effectively treat this as a DR and skip
// language version checks.
- emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, size, successOrder,
- failureOrder, scope);
+ emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
+ size, successOrder, failureOrder, scope);
return;
}
@@ -587,8 +598,8 @@ static void emitAtomicCmpXchgFailureSet(CIRGenFunction &cgf, AtomicExpr *e,
// which seems reasonable. Also, 'relaxed' being the default behavior
// is also probably the least harmful.
emitDefaultCaseLabel(cgf.getBuilder(), atomicLoc);
- emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, size,
- successOrder, cir::MemOrder::Relaxed, scope);
+ emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
+ size, successOrder, cir::MemOrder::Relaxed, scope);
cgf.getBuilder().createBreak(atomicLoc);
cgf.getBuilder().setInsertionPointToEnd(switchBlock);
@@ -596,17 +607,17 @@ static void emitAtomicCmpXchgFailureSet(CIRGenFunction &cgf, AtomicExpr *e,
// case cir::MemOrder::Acquire:
emitMemOrderCaseLabel(cgf.getBuilder(), loc, failureOrderVal.getType(),
{cir::MemOrder::Consume, cir::MemOrder::Acquire});
- emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, size,
- successOrder, cir::MemOrder::Acquire, scope);
+ emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
+ size, successOrder, cir::MemOrder::Acquire, scope);
cgf.getBuilder().createBreak(atomicLoc);
cgf.getBuilder().setInsertionPointToEnd(switchBlock);
// case cir::MemOrder::SequentiallyConsistent:
emitMemOrderCaseLabel(cgf.getBuilder(), loc, failureOrderVal.getType(),
{cir::MemOrder::SequentiallyConsistent});
- emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, size,
- successOrder, cir::MemOrder::SequentiallyConsistent,
- scope);
+ emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
+ size, successOrder,
+ cir::MemOrder::SequentiallyConsistent, scope);
cgf.getBuilder().createBreak(atomicLoc);
cgf.getBuilder().setInsertionPointToEnd(switchBlock);
@@ -619,8 +630,9 @@ static void emitAtomicCmpXchgFailureSet(CIRGenFunction &cgf, AtomicExpr *e,
// emitAtomicCmpXchgFailureSet 2x).
static void emitAtomicCmpXchgFailureSetCheckWeak(
CIRGenFunction &cgf, AtomicExpr *e, Expr *isWeakExpr, Address dest,
- Address ptr, Address val1, Address val2, Expr *failureOrderExpr,
- uint64_t size, cir::MemOrder successOrder, cir::SyncScopeKind scope) {
+ Address ptr, Address val1, Address val2, Address expectedResult,
+ Expr *failureOrderExpr, uint64_t size, cir::MemOrder successOrder,
+ cir::SyncScopeKind scope) {
mlir::Value isWeakVal = cgf.emitScalarExpr(isWeakExpr);
// The AST seems to be inserting a 'bool' cast (even in C mode) here, so we'll
// just emit it like a scalar.
@@ -634,21 +646,22 @@ static void emitAtomicCmpXchgFailureSetCheckWeak(
cgf.getBuilder(), atomicLoc, isWeakVal, /*elseRegion=*/true,
[&](mlir::OpBuilder &b, mlir::Location loc) {
emitAtomicCmpXchgFailureSet(cgf, e, /*isWeak=*/true, dest, ptr, val1,
- val2, failureOrderExpr, size, successOrder,
- scope);
+ val2, expectedResult, failureOrderExpr,
+ size, successOrder, scope);
cgf.getBuilder().createYield(atomicLoc);
},
[&](mlir::OpBuilder &b, mlir::Location loc) {
emitAtomicCmpXchgFailureSet(cgf, e, /*isWeak=*/false, dest, ptr, val1,
- val2, failureOrderExpr, size, successOrder,
- scope);
+ val2, expectedResult, failureOrderExpr,
+ size, successOrder, scope);
cgf.getBuilder().createYield(atomicLoc);
});
}
static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
Address ptr, Address val1, Address val2,
- Expr *isWeakExpr, Expr *failureOrderExpr, int64_t size,
+ Address expectedResult, Expr *isWeakExpr,
+ Expr *failureOrderExpr, int64_t size,
cir::MemOrder order, cir::SyncScopeKind scope) {
assert(!cir::MissingFeatures::atomicSyncScopeID());
llvm::StringRef opName;
@@ -673,14 +686,16 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
emitAtomicCmpXchgFailureSet(cgf, expr, /*isWeak=*/false, dest, ptr, val1,
- val2, failureOrderExpr, size, order, scope);
+ val2, expectedResult, failureOrderExpr, size,
+ order, scope);
return;
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
emitAtomicCmpXchgFailureSet(cgf, expr, /*isWeak=*/true, dest, ptr, val1,
- val2, failureOrderExpr, size, order, scope);
+ val2, expectedResult, failureOrderExpr, size,
+ order, scope);
return;
case AtomicExpr::AO__atomic_compare_exchange:
@@ -690,11 +705,12 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
bool isWeak = false;
if (isWeakExpr->EvaluateAsBooleanCondition(isWeak, cgf.getContext())) {
emitAtomicCmpXchgFailureSet(cgf, expr, isWeak, dest, ptr, val1, val2,
- failureOrderExpr, size, order, scope);
+ expectedResult, failureOrderExpr, size, order,
+ scope);
} else {
- emitAtomicCmpXchgFailureSetCheckWeak(cgf, expr, isWeakExpr, dest, ptr,
- val1, val2, failureOrderExpr, size,
- order, scope);
+ emitAtomicCmpXchgFailureSetCheckWeak(
+ cgf, expr, isWeakExpr, dest, ptr, val1, val2, expectedResult,
+ failureOrderExpr, size, order, scope);
}
return;
}
@@ -965,15 +981,16 @@ static cir::SyncScopeKind convertSyncScopeToCIR(CIRGenFunction &cgf,
static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
Address ptr, Address val1, Address val2,
- Expr *isWeakExpr, Expr *failureOrderExpr, int64_t size,
+ Address expectedResult, Expr *isWeakExpr,
+ Expr *failureOrderExpr, int64_t size,
cir::MemOrder order,
const std::optional<Expr::EvalResult> &scopeConst,
mlir::Value scopeValue) {
std::unique_ptr<AtomicScopeModel> scopeModel = expr->getScopeModel();
if (!scopeModel) {
- emitAtomicOp(cgf, expr, dest, ptr, val1, val2, isWeakExpr, failureOrderExpr,
- size, order, cir::SyncScopeKind::System);
+ emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr,
+ failureOrderExpr, size, order, cir::SyncScopeKind::System);
return;
}
@@ -981,8 +998,8 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
cir::SyncScopeKind mappedScope = convertSyncScopeToCIR(
cgf, expr->getScope()->getSourceRange(),
scopeModel->map(scopeConst->Val.getInt().getZExtValue()));
- emitAtomicOp(cgf, expr, dest, ptr, val1, val2, isWeakExpr, failureOrderExpr,
- size, order, mappedScope);
+ emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr,
+ failureOrderExpr, size, order, mappedScope);
return;
}
@@ -1002,8 +1019,8 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
cir::SyncScopeKind fallbackScope = convertSyncScopeToCIR(
cgf, expr->getScope()->getSourceRange(), scopeModel->map(fallback));
emitDefaultCaseLabel(builder, loc);
- emitAtomicOp(cgf, expr, dest, ptr, val1, val2, isWeakExpr,
- failureOrderExpr, size, order, fallbackScope);
+ emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult,
+ isWeakExpr, failureOrderExpr, size, order, fallbackScope);
builder.createBreak(loc);
builder.setInsertionPointToEnd(switchBlock);
@@ -1022,8 +1039,8 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
insertPoint);
builder.restoreInsertionPoint(insertPoint);
- emitAtomicOp(cgf, expr, dest, ptr, val1, val2, isWeakExpr,
- failureOrderExpr, size, order, cirScope);
+ emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult,
+ isWeakExpr, failureOrderExpr, size, order, cirScope);
builder.createBreak(loc);
builder.setInsertionPointToEnd(switchBlock);
}
@@ -1589,6 +1606,10 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
LValue atomicValue = makeAddrLValue(ptr, atomicTy);
AtomicInfo atomics(*this, atomicValue, loc);
+ // Save val1's address before it is (possibly) converted to a temporary
+ // sized to the full atomic width below.
+ Address originalVal1 = val1;
+
if (shouldCastToIntPtrTy) {
ptr = atomics.castToAtomicIntPointer(ptr);
if (val1.isValid())
@@ -1642,8 +1663,8 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
e->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
auto emitAtomicOpCallBackFn = [&](cir::MemOrder memOrder) {
- emitAtomicOp(*this, e, dest, ptr, val1, val2, isWeakExpr, orderFailExpr,
- size, memOrder, scopeConst, scope);
+ emitAtomicOp(*this, e, dest, ptr, val1, val2, originalVal1, isWeakExpr,
+ orderFailExpr, size, memOrder, scopeConst, scope);
};
emitAtomicExprWithMemOrder(e->getOrder(), isStore, isLoad, /*isFence*/ false,
emitAtomicOpCallBackFn);
diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c
index d2fb28116e7bd..bd6e49749f90f 100644
--- a/clang/test/CIR/CodeGen/atomic.c
+++ b/clang/test/CIR/CodeGen/atomic.c
@@ -3979,17 +3979,17 @@ void store_atomic_different_size(S a) {
__c11_atomic_store(&b, a, __ATOMIC_SEQ_CST);
// CIR: %[[A_ADDR:.*]] = cir.alloca "a" {{.*}} init : !cir.ptr<!rec_S>
- // CIR: %[[B_ADDR:.*]] = cir.alloca "b" {{.*}} : !cir.ptr<!rec_anon_struct1>
+ // CIR: %[[B_ADDR:.*]] = cir.alloca "b" {{.*}} : !cir.ptr<!rec_anon_struct2>
// CIR: %[[A_ATOMIC_TMP_ADDR:.*]] = cir.alloca ".atomictmp" {{.*}} : !cir.ptr<!rec_S>
- // CIR: %[[ATOMIC_TMP_ADDR:.*]] = cir.alloca "atomic-temp" {{.*}} : !cir.ptr<!rec_anon_struct1>
+ // CIR: %[[ATOMIC_TMP_ADDR:.*]] = cir.alloca "atomic-temp" {{.*}} : !cir.ptr<!rec_anon_struct2>
// CIR: cir.store %[[A:.*]], %[[A_ADDR]] : !rec_S, !cir.ptr<!rec_S>
// CIR: cir.copy %[[A_ADDR]] {{.*}} to %[[A_ATOMIC_TMP_ADDR]] {{.*}} : !cir.ptr<!rec_S>
- // CIR: %[[B_VOID_PTR:.*]] = cir.cast bitcast %[[B_ADDR]] : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!u32i>
+ // CIR: %[[B_VOID_PTR:.*]] = cir.cast bitcast %[[B_ADDR]] : !cir.ptr<!rec_anon_struct2> -> !cir.ptr<!u32i>
// CIR: %[[CONST_0:.*]] = cir.const #cir.int<0> : !u8i
// CIR: %[[MEMSET_SIZE:.*]] = cir.const #cir.int<4> : !u64i
// CIR: %[[A_VOID_PTR:.*]] = cir.cast bitcast %[[A_ATOMIC_TMP_ADDR]] : !cir.ptr<!rec_S> -> !cir.ptr<!void>
- // CIR: cir.libc.memset %[[MEMSET_SIZE]] bytes at %[[A_VOID_PTR]] {{.*}} to %[[CONST_0]] : !cir.ptr<!void>, !u8i, !u64i
- // CIR: %[[ATOMIC_TMP:.*]] = cir.cast bitcast %[[ATOMIC_TMP_ADDR]] : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!void>
+ // CIR: %[[ATOMIC_TMP:.*]] = cir.cast bitcast %[[ATOMIC_TMP_ADDR]] : !cir.ptr<!rec_anon_struct2> -> !cir.ptr<!void>
+ // CIR: cir.libc.memset %[[MEMSET_SIZE]] bytes at %[[ATOMIC_TMP]] {{.*}} to %[[CONST_0]] : !cir.ptr<!void>, !u8i, !u64i
// CIR: %[[MEMCPY_SIZE:.*]] = cir.const #cir.int<3> : !u64i
// CIR: cir.libc.memcpy %[[MEMCPY_SIZE]] bytes from %[[A_VOID_PTR]] align(1) to %[[ATOMIC_TMP]] align(4) : !u64i, !cir.ptr<!void> -> !cir.ptr<!void>
// CIR: %[[ATOMIC_TMP_U32:.*]] = cir.cast bitcast %[[ATOMIC_TMP]] : !cir.ptr<!void> -> !cir.ptr<!u32i>
@@ -4005,7 +4005,7 @@ void store_atomic_different_size(S a) {
// LLVM: %[[ATOMIC_TMP_ADDR:.*]] = alloca { %struct.S, [1 x i8] }, align 4
// LLVM: store %struct.S %[[A]], ptr %[[A_ADDR]], align 1
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[A_ATOMIC_TMP_ADDR]], ptr align 1 %[[A_ADDR]], i64 3, i1 false)
- // LLVM: call void @llvm.memset.p0.i64(ptr align 1 %[[A_ATOMIC_TMP_ADDR]], i8 0, i64 4, i1 false)
+ // LLVM: call void @llvm.memset.p0.i64(ptr align 4 %[[ATOMIC_TMP_ADDR]], i8 0, i64 4, i1 false)
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[ATOMIC_TMP_ADDR]], ptr align 1 %[[A_ATOMIC_TMP_ADDR]], i64 3, i1 false)
// LLVM: %[[ATOMIC_TMP:.*]] = load i32, ptr %[[ATOMIC_TMP_ADDR]], align 4
// LLVM: store atomic i32 %[[ATOMIC_TMP]], ptr %[[B_ADDR]] seq_cst, align 4
@@ -4237,3 +4237,63 @@ float atomic_float_post_dec(_Atomic(float) *p) { return (*p)--; }
// OGCG-LABEL: @atomic_float_post_dec
// OGCG: %[[OLD:.+]] = atomicrmw fsub ptr %{{.+}}, float 1.000000e+00 seq_cst, align 4
// OGCG: ret float %[[OLD]]
+
+struct S2 { char c; int i; char c2; };
+
+_Bool atomic_cmpxchg_struct_padding(_Atomic(struct S2) *ptr, struct S2 *expected, struct S2 *desired) {
+ return __c11_atomic_compare_exchange_strong(ptr, expected, *desired, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+// CIR-LABEL: @atomic_cmpxchg_struct_padding
+// CIR: %[[DESIRED_TMP:.+]] = cir.alloca ".atomictmp" {{.*}} : !cir.ptr<!rec_S2>
+// CIR: %[[EXPECTED_ATOMIC_TMP_ADDR:.+]] = cir.alloca "atomic-temp" {{.*}} : !cir.ptr<!rec_anon_struct{{[0-9]*}}>
+// CIR: %[[OLD_TMP:.+]] = cir.alloca "old.tmp" {{.*}} : !cir.ptr<!u128i>
+// CIR: %[[PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_anon_struct{{[0-9]*}}>>, !cir.ptr<!rec_anon_struct{{[0-9]*}}>
+// CIR: %[[EXPECTED:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_S2>>, !cir.ptr<!rec_S2>
+// CIR: %[[DESIRED:.+]] = cir.load deref align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_S2>>, !cir.ptr<!rec_S2>
+// CIR: cir.copy %[[DESIRED]] {{.*}} to %[[DESIRED_TMP]] {{.*}} : !cir.ptr<!rec_S2>
+// CIR: %[[EXPECTED_VOID:.+]] = cir.cast bitcast %[[EXPECTED]] : !cir.ptr<!rec_S2> -> !cir.ptr<!void>
+// CIR: %[[EXPECTED_ATOMIC_TMP:.+]] = cir.cast bitcast %[[EXPECTED_ATOMIC_TMP_ADDR]] : !cir.ptr<!rec_anon_struct{{[0-9]*}}> -> !cir.ptr<!void>
+
+// CIR-NOT: cir.libc.memset {{.*}} bytes at %[[EXPECTED_VOID]]
+// CIR: cir.libc.memset %{{.+}} bytes at %[[EXPECTED_ATOMIC_TMP]] {{.*}} to %{{.+}} : !cir.ptr<!void>, !u8i, !u64i
+
+// CIR: cir.libc.memcpy %{{.+}} bytes from %[[EXPECTED_VOID]] {{.*}} to %[[EXPECTED_ATOMIC_TMP]] {{.*}} : !u64i, !cir.ptr<!void> -> !cir.ptr<!void>
+// CIR: %old, %success = cir.atomic.cmpxchg success(seq_cst) failure(seq_cst) syncscope(system) %{{.+}}, %{{.+}}, %{{.+}} align(16) : (!cir.ptr<!u128i>, !u128i, !u128i) -> (!u128i, !cir.bool)
+// CIR: cir.if %{{.+}} {
+// CIR: cir.store align(16) %old, %[[OLD_TMP]] : !u128i, !cir.ptr<!u128i>
+// CIR: %[[OLD_TMP_VOID:.+]] = cir.cast bitcast %[[OLD_TMP]] : !cir.ptr<!u128i> -> !cir.ptr<!void>
+// back, and it is written into "expected" itself, not into a temporary.
+// CIR: %[[EXPECTED_VOID2:.+]] = cir.cast bitcast %[[EXPECTED]] : !cir.ptr<!rec_S2> -> !cir.ptr<!void>
+// CIR: cir.libc.memcpy %{{.+}} bytes from %[[OLD_TMP_VOID]] {{.*}} to %[[EXPECTED_VOID2]] {{.*}} : !u64i, !cir.ptr<!void> -> !cir.ptr<!void>
+// CIR: }
+
+// LLVM-LABEL: @atomic_cmpxchg_struct_padding
+// LLVM: %[[DESIRED_TMP:.+]] = alloca %struct.S2, align 4
+// LLVM: %[[EXPECTED_TMP:.+]] = alloca { %struct.S2, [4 x i8] }, align 16
+// LLVM: %[[OLD_TMP:.+]] = alloca i128, align 16
+// LLVM: %[[PTR:.+]] = load ptr, ptr %{{.+}}, align 8
+// LLVM: %[[EXPECTED:.+]] = load ptr, ptr %{{.+}}, align 8
+// LLVM: %[[DESIRED:.+]] = load ptr, ptr %{{.+}}, align 8
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[DESIRED_TMP]], ptr align 4 %[[DESIRED]], i64 12, i1 false)
+// LLVM: call void @llvm.memset.p0.i64(ptr align 16 %[[EXPECTED_TMP]], i8 0, i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[EXPECTED_TMP]], ptr align 4 %[[EXPECTED]], i64 12, i1 false)
+// LLVM: %[[CMPXCHG:.+]] = cmpxchg ptr %[[PTR]], i128 %{{.+}}, i128 %{{.+}} seq_cst seq_cst, align 16
+// LLVM: %[[OLD:.+]] = extractvalue { i128, i1 } %[[CMPXCHG]], 0
+// LLVM: store i128 %[[OLD]], ptr %[[OLD_TMP]], align 16
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[EXPECTED]], ptr align 16 %[[OLD_TMP]], i64 12, i1 false)
+
+// OGCG-LABEL: @atomic_cmpxchg_struct_padding
+// OGCG: %[[DESIRED_TMP:.+]] = alloca %struct.S2, align 4
+// OGCG: %[[EXPECTED_TMP:.+]] = alloca { %struct.S2, [4 x i8] }, align 16
+// OGCG: %[[OLD_TMP:.+]] = alloca i128, align 16
+// OGCG: %[[PTR:.+]] = load ptr, ptr %{{.+}}, align 8
+// OGCG: %[[EXPECTED:.+]] = load ptr, ptr %{{.+}}, align 8
+// OGCG: %[[DESIRED:.+]] = load ptr, ptr %{{.+}}, align 8
+// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[DESIRED_TMP]], ptr align 4 %[[DESIRED]], i64 12, i1 false)
+// OGCG: call void @llvm.memset.p0.i64(ptr align 16 %[[EXPECTED_TMP]], i8 0, i64 16, i1 false)
+// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[EXPECTED_TMP]], ptr align 4 %[[EXPECTED]], i64 12, i1 false)
+// OGCG: %[[CMPXCHG:.+]] = cmpxchg ptr %[[PTR]], i128 %{{.+}}, i128 %{{.+}} seq_cst seq_cst, align 16
+// OGCG: %[[OLD:.+]] = extractvalue { i128, i1 } %[[CMPXCHG]], 0
+// OGCG: store i128 %[[OLD]], ptr %[[OLD_TMP]], align 16
+// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[EXPECTED]], ptr align 16 %[[OLD_TMP]], i64 12, i1 false)
>From 2ea88b6638fdad750afa67c6da64e05324b9840a Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane at nvidia.com>
Date: Mon, 28 Sep 2026 10:44:27 -0700
Subject: [PATCH 2/3] Fixup test that used SSA value
---
clang/test/CIR/CodeGen/atomic.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c
index bd6e49749f90f..3481e6a56a842 100644
--- a/clang/test/CIR/CodeGen/atomic.c
+++ b/clang/test/CIR/CodeGen/atomic.c
@@ -4259,9 +4259,9 @@ _Bool atomic_cmpxchg_struct_padding(_Atomic(struct S2) *ptr, struct S2 *expected
// CIR: cir.libc.memset %{{.+}} bytes at %[[EXPECTED_ATOMIC_TMP]] {{.*}} to %{{.+}} : !cir.ptr<!void>, !u8i, !u64i
// CIR: cir.libc.memcpy %{{.+}} bytes from %[[EXPECTED_VOID]] {{.*}} to %[[EXPECTED_ATOMIC_TMP]] {{.*}} : !u64i, !cir.ptr<!void> -> !cir.ptr<!void>
-// CIR: %old, %success = cir.atomic.cmpxchg success(seq_cst) failure(seq_cst) syncscope(system) %{{.+}}, %{{.+}}, %{{.+}} align(16) : (!cir.ptr<!u128i>, !u128i, !u128i) -> (!u128i, !cir.bool)
+// CIR: %[[OLD:.*]], %[[SUCCESS:.*]] = cir.atomic.cmpxchg success(seq_cst) failure(seq_cst) syncscope(system) %{{.+}}, %{{.+}}, %{{.+}} align(16) : (!cir.ptr<!u128i>, !u128i, !u128i) -> (!u128i, !cir.bool)
// CIR: cir.if %{{.+}} {
-// CIR: cir.store align(16) %old, %[[OLD_TMP]] : !u128i, !cir.ptr<!u128i>
+// CIR: cir.store align(16) %[[OLD]], %[[OLD_TMP]] : !u128i, !cir.ptr<!u128i>
// CIR: %[[OLD_TMP_VOID:.+]] = cir.cast bitcast %[[OLD_TMP]] : !cir.ptr<!u128i> -> !cir.ptr<!void>
// back, and it is written into "expected" itself, not into a temporary.
// CIR: %[[EXPECTED_VOID2:.+]] = cir.cast bitcast %[[EXPECTED]] : !cir.ptr<!rec_S2> -> !cir.ptr<!void>
>From 0f15c8bb2f5b23df7b00cdad27544d32d371f9dd Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane at nvidia.com>
Date: Mon, 28 Sep 2026 10:46:05 -0700
Subject: [PATCH 3/3] Added missing-features-asserts for debug info
---
clang/lib/CIR/CodeGen/CIRGenAtomic.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
index fc47e732bc8a0..d974f0792d481 100644
--- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
@@ -516,10 +516,12 @@ static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,
Address storeAddr = expectedResult.withElementType(
builder, cmpxchg.getOld().getType());
builder.createStore(loc, cmpxchg.getOld(), storeAddr);
+ assert(!MissingFeatures::generateDebugInfo());
} else {
Address oldTmp = cgf.createTempAlloca(
cmpxchg.getOld().getType(), ptr.getAlignment(), loc, "old.tmp");
builder.createStore(loc, cmpxchg.getOld(), oldTmp);
+ assert(!MissingFeatures::generateDebugInfo());
Address oldTmpVoid = oldTmp.withElementType(builder, cgf.cgm.voidTy);
Address expectedVoid =
More information about the cfe-commits
mailing list