[llvm-branch-commits] [clang] [HLSL] Add `InterlockedCompareExchange` function and resource methods (PR #222165)
Joshua Batista via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 8 14:51:07 PDT 2026
https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/222165
This PR adds the `InterlockedCompareExchange` standalone function and resource
methods. The operation lowers to the same `cmpxchg monotonic` as
`InterlockedCompareStore`.
`InterlockedCompareExchange` reports the previous value, so an `extractvalue`
reads it from the `cmpxchg` result and stores it through the `original_value`
reference parameter.
The PR also adds the 64-bit `InterlockedCompareExchange64` methods, which DXIL
gates on shader model 6.6.
Fixes: https://github.com/llvm/llvm-project/issues/99130
Assisted by: Github Copilot
>From 83fd470639b181bc436c67b7f5e716de72c3032f Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Tue, 8 Sep 2026 13:06:24 -0700
Subject: [PATCH] First attempt implementing InterlockedCompareExchange
---
clang/include/clang/Basic/Builtins.td | 9 ++
clang/lib/CodeGen/CGHLSLBuiltins.cpp | 25 +++-
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 32 +++--
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h | 5 +-
clang/lib/Sema/HLSLExternalSemaSource.cpp | 67 ++++++----
clang/lib/Sema/SemaHLSL.cpp | 36 ++++--
.../builtins/InterlockedCompareExchange.hlsl | 70 +++++++++++
.../builtins/RWBuffer-Interlocked.hlsl | 5 +
...ressBuffer-InterlockedCompareExchange.hlsl | 44 +++++++
...ressBuffer-InterlockedCompareExchange.hlsl | 35 ++++++
...nterlockedCompareExchange-sm65-errors.hlsl | 30 +++++
.../InterlockedCompareExchange-errors.hlsl | 114 ++++++++++++++++++
12 files changed, 420 insertions(+), 52 deletions(-)
create mode 100644 clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl
create mode 100644 clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl
create mode 100644 clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index de8b600db6763..c07cffbe4ebae 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5557,6 +5557,15 @@ def HLSLInterlockedAnd : LangBuiltin<"HLSL_LANG"> {
let Prototype = "void (...)";
}
+def HLSLInterlockedCompareExchange : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_interlocked_compare_exchange"];
+ // SemaHLSL checks these arguments itself. Custom type checking also stops
+ // the default variadic promotion, which would otherwise widen a float
+ // `dest` to double and misreport its type in the diagnostic.
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void (...)";
+}
+
def HLSLInterlockedCompareStore : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_interlocked_compare_store"];
// SemaHLSL checks these arguments itself. Custom type checking also stops
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index ef379aca49a5f..80835f6f27f2f 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -348,18 +348,30 @@ static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
}
// InterlockedCompareStore(dest, compare_value, value) stores `value` only when
-// `dest` holds `compare_value`. It reports nothing, so the `cmpxchg` result is
-// unused. DXILResourceAccess and the SPIR-V selector both match `cmpxchg`.
-static Value *handleInterlockedCompareStore(CodeGenFunction &CGF,
- const CallExpr *E) {
+// `dest` holds `compare_value`. InterlockedCompareExchange takes the same
+// arguments and additionally reports the previous value. DXILResourceAccess
+// and the SPIR-V selector both match `cmpxchg`.
+static Value *handleInterlockedCompareOp(CodeGenFunction &CGF,
+ const CallExpr *E) {
LValue DestLV = CGF.EmitLValue(E->getArg(0));
Address DestAddr = DestLV.getAddress();
Value *Compare = CGF.EmitScalarExpr(E->getArg(1));
Value *Val = CGF.EmitScalarExpr(E->getArg(2));
- return CGF.Builder.CreateAtomicCmpXchg(
+ Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
DestAddr, Compare, Val, llvm::AtomicOrdering::Monotonic,
llvm::AtomicOrdering::Monotonic, getHLSLAtomicScope(CGF, DestLV));
+
+ // Compare-store reports nothing, so it leaves the `cmpxchg` result unused.
+ if (E->getNumArgs() < 4)
+ return Pair;
+
+ // `cmpxchg` yields a { previous value, success } pair. HLSL reports only the
+ // previous value, through the `original_value` reference parameter.
+ Value *Original = CGF.Builder.CreateExtractValue(Pair, 0);
+ LValue OrigLV = CGF.EmitLValue(E->getArg(3));
+ CGF.EmitStoreThroughLValue(RValue::get(Original), OrigLV);
+ return Original;
}
static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,
@@ -1503,8 +1515,9 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
case Builtin::BI__builtin_hlsl_interlocked_and: {
return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::And);
}
+ case Builtin::BI__builtin_hlsl_interlocked_compare_exchange:
case Builtin::BI__builtin_hlsl_interlocked_compare_store: {
- return handleInterlockedCompareStore(*this, E);
+ return handleInterlockedCompareOp(*this, E);
}
case Builtin::BI__builtin_hlsl_interlocked_exchange: {
return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::Xchg);
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
index 16bc77c5ea127..3c374f8649c55 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
@@ -1753,7 +1753,11 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() {
"__builtin_hlsl_interlocked_add");
addByteAddressBufferInterlockedMethod("InterlockedAnd", AST.UnsignedIntTy,
"__builtin_hlsl_interlocked_and");
- addByteAddressBufferInterlockedCompareStoreMethod(
+ addByteAddressBufferInterlockedCompareMethod(
+ "InterlockedCompareExchange", AST.UnsignedIntTy,
+ "__builtin_hlsl_interlocked_compare_exchange",
+ /*WithOriginalValue=*/true);
+ addByteAddressBufferInterlockedCompareMethod(
"InterlockedCompareStore", AST.UnsignedIntTy,
"__builtin_hlsl_interlocked_compare_store");
addByteAddressBufferInterlockedMethod(
@@ -1791,7 +1795,11 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() {
addByteAddressBufferInterlockedMethod("InterlockedAnd64",
AST.UnsignedLongTy,
"__builtin_hlsl_interlocked_and");
- addByteAddressBufferInterlockedCompareStoreMethod(
+ addByteAddressBufferInterlockedCompareMethod(
+ "InterlockedCompareExchange64", AST.UnsignedLongTy,
+ "__builtin_hlsl_interlocked_compare_exchange",
+ /*WithOriginalValue=*/true);
+ addByteAddressBufferInterlockedCompareMethod(
"InterlockedCompareStore64", AST.UnsignedLongTy,
"__builtin_hlsl_interlocked_compare_store");
addByteAddressBufferInterlockedMethod(
@@ -2722,14 +2730,16 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethod(
}
BuiltinTypeDeclBuilder &
-BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedCompareStoreMethod(
- StringRef MethodName, QualType ValueTy, StringRef BuiltinName) {
+BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedCompareMethod(
+ StringRef MethodName, QualType ValueTy, StringRef BuiltinName,
+ bool WithOriginalValue) {
assert(!Record->isCompleteDefinition() && "record is already complete");
ASTContext &AST = SemaRef.getASTContext();
using PH = BuiltinTypeMethodBuilder::PlaceHolder;
- // Compare-store reports nothing, so it has a single overload. It reaches the
- // buffer slot the same way as the other interlocked methods.
+ // Compare-store reports nothing and compare-exchange always reports the
+ // previous value, so each has a single overload. Both reach the buffer slot
+ // the same way as the other interlocked methods.
QualType AddrSpaceElemTy =
AST.getAddrSpaceQualType(ValueTy, LangAS::hlsl_device);
QualType ElemPtrTy = AST.getPointerType(AddrSpaceElemTy);
@@ -2738,10 +2748,16 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedCompareStoreMethod(
MMB.addParam("Offset", AST.UnsignedIntTy)
.addParam("CompareValue", ValueTy)
.addParam("Value", ValueTy);
+ if (WithOriginalValue)
+ MMB.addParam("OriginalValue", ValueTy, HLSLParamModifierAttr::Keyword_out);
MMB.callBuiltin("__builtin_hlsl_resource_getpointer_typed", ElemPtrTy,
PH::Handle, PH::_0, ValueTy)
- .dereference(PH::LastStmt)
- .callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2);
+ .dereference(PH::LastStmt);
+ if (WithOriginalValue)
+ MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2,
+ PH::_3);
+ else
+ MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2);
MMB.finalize();
return *this;
}
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
index 92260868c2719..aa77dc7aa7164 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
@@ -151,8 +151,9 @@ class BuiltinTypeDeclBuilder {
addByteAddressBufferInterlockedMethod(StringRef MethodName, QualType ValueTy,
StringRef BuiltinName,
bool RequiresOriginalValue = false);
- BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedCompareStoreMethod(
- StringRef MethodName, QualType ValueTy, StringRef BuiltinName);
+ BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedCompareMethod(
+ StringRef MethodName, QualType ValueTy, StringRef BuiltinName,
+ bool WithOriginalValue = false);
BuiltinTypeDeclBuilder &addAppendMethod();
BuiltinTypeDeclBuilder &addConsumeMethod();
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index 443b1d94757fc..13539c421b8e0 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -813,12 +813,13 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
}
// Shapes of the synthesized atomic overloads. The read-modify-write operations
-// can report the previous value through a trailing reference. Every argument
-// of compare-store is an input.
+// can report the previous value through a trailing reference. Compare-store
+// takes only inputs; compare-exchange adds the trailing reference.
enum class AtomicOverloadShape {
Binary, // (dest, value)
BinaryWithOriginal, // (dest, value, original_value)
CompareStore, // (dest, compare_value, value)
+ CompareExchange, // (dest, compare_value, value, original_value)
};
// Build a single overload of an HLSL atomic intrinsic in the hlsl namespace.
@@ -834,20 +835,32 @@ static void buildAtomicOverload(Sema &S, NamespaceDecl *NS, StringRef FuncName,
AST.getLValueReferenceType(AST.getAddrSpaceQualType(ElemTy, DestAS));
QualType OrigRefTy = AST.getLValueReferenceType(ElemTy);
- SmallVector<QualType, 3> ParamTypes = {DestTy, ElemTy};
- if (Shape == AtomicOverloadShape::BinaryWithOriginal)
+ SmallVector<QualType, 4> ParamTypes = {DestTy, ElemTy};
+ switch (Shape) {
+ case AtomicOverloadShape::Binary:
+ break;
+ case AtomicOverloadShape::BinaryWithOriginal:
ParamTypes.push_back(OrigRefTy);
- else if (Shape == AtomicOverloadShape::CompareStore)
+ break;
+ case AtomicOverloadShape::CompareStore:
+ ParamTypes.push_back(ElemTy);
+ break;
+ case AtomicOverloadShape::CompareExchange:
ParamTypes.push_back(ElemTy);
+ ParamTypes.push_back(OrigRefTy);
+ break;
+ }
- // The loop below stops at the end of ParamTypes, so a two-argument overload
- // ignores the trailing name.
+ // The loop below stops at the end of ParamTypes, so a shorter overload
+ // ignores the trailing names.
constexpr const char *BinaryNames[] = {"dest", "value", "original_value"};
- constexpr const char *CompareStoreNames[] = {"dest", "compare_value",
- "value"};
- ArrayRef<const char *> ParamNames = Shape == AtomicOverloadShape::CompareStore
- ? CompareStoreNames
- : BinaryNames;
+ constexpr const char *CompareNames[] = {"dest", "compare_value", "value",
+ "original_value"};
+ const bool IsCompare = Shape == AtomicOverloadShape::CompareStore ||
+ Shape == AtomicOverloadShape::CompareExchange;
+ ArrayRef<const char *> ParamNames = IsCompare
+ ? ArrayRef<const char *>(CompareNames)
+ : ArrayRef<const char *>(BinaryNames);
FunctionProtoType::ExtProtoInfo EPI;
QualType FuncTy = AST.getFunctionType(AST.VoidTy, ParamTypes, EPI);
@@ -861,7 +874,7 @@ static void buildAtomicOverload(Sema &S, NamespaceDecl *NS, StringRef FuncName,
SC_Extern, /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
/*hasWrittenPrototype=*/true);
- SmallVector<ParmVarDecl *, 3> ParmDecls;
+ SmallVector<ParmVarDecl *, 4> ParmDecls;
unsigned I = 0;
for (auto [ParamType, ParamName] : llvm::zip(ParamTypes, ParamNames)) {
IdentifierInfo &PII = AST.Idents.get(ParamName, tok::TokenKind::identifier);
@@ -909,12 +922,14 @@ static void defineHLSLInterlockedFunc(Sema &S, NamespaceDecl *NS,
}
}
-// Synthesize the InterlockedCompareStore overload set: {int, uint, int64_t,
-// uint64_t} x {groupshared, device}. The operation reports nothing, so it has
-// a single arity. Float compare-store is a separate intrinsic.
-static void defineHLSLInterlockedCompareStoreFunc(Sema &S, NamespaceDecl *NS,
- StringRef FuncName,
- StringRef BuiltinName) {
+// Synthesize the compare-and-swap overload sets: {int, uint, int64_t,
+// uint64_t} x {groupshared, device}. Compare-store reports nothing and
+// compare-exchange always reports the previous value, so each has a single
+// arity. The float forms are separate intrinsics.
+static void defineHLSLInterlockedCompareFunc(Sema &S, NamespaceDecl *NS,
+ StringRef FuncName,
+ StringRef BuiltinName,
+ AtomicOverloadShape Shape) {
ASTContext &AST = S.getASTContext();
// HLSL: int64_t == long, uint64_t == unsigned long (see hlsl_basic_types.h).
QualType Elems[] = {AST.IntTy, AST.UnsignedIntTy, AST.LongTy,
@@ -922,8 +937,7 @@ static void defineHLSLInterlockedCompareStoreFunc(Sema &S, NamespaceDecl *NS,
for (QualType ElemTy : Elems)
for (LangAS AS : {LangAS::hlsl_groupshared, LangAS::hlsl_device})
- buildAtomicOverload(S, NS, FuncName, BuiltinName, ElemTy, AS,
- AtomicOverloadShape::CompareStore);
+ buildAtomicOverload(S, NS, FuncName, BuiltinName, ElemTy, AS, Shape);
}
void HLSLExternalSemaSource::defineHLSLAtomicIntrinsics() {
@@ -931,9 +945,14 @@ void HLSLExternalSemaSource::defineHLSLAtomicIntrinsics() {
"__builtin_hlsl_interlocked_add");
defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedAnd",
"__builtin_hlsl_interlocked_and");
- defineHLSLInterlockedCompareStoreFunc(
- *SemaPtr, HLSLNamespace, "InterlockedCompareStore",
- "__builtin_hlsl_interlocked_compare_store");
+ defineHLSLInterlockedCompareFunc(
+ *SemaPtr, HLSLNamespace, "InterlockedCompareExchange",
+ "__builtin_hlsl_interlocked_compare_exchange",
+ AtomicOverloadShape::CompareExchange);
+ defineHLSLInterlockedCompareFunc(*SemaPtr, HLSLNamespace,
+ "InterlockedCompareStore",
+ "__builtin_hlsl_interlocked_compare_store",
+ AtomicOverloadShape::CompareStore);
defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedExchange",
"__builtin_hlsl_interlocked_exchange",
/*RequiresOriginalValue=*/true,
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 2694ddcc2e306..bde8e806b5b82 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -4712,6 +4712,7 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
}
case Builtin::BI__builtin_hlsl_interlocked_add:
case Builtin::BI__builtin_hlsl_interlocked_and:
+ case Builtin::BI__builtin_hlsl_interlocked_compare_exchange:
case Builtin::BI__builtin_hlsl_interlocked_compare_store:
case Builtin::BI__builtin_hlsl_interlocked_exchange:
case Builtin::BI__builtin_hlsl_interlocked_max:
@@ -4729,10 +4730,17 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
// argument is an input rather than an output.
const bool IsCompareStore =
BuiltinID == Builtin::BI__builtin_hlsl_interlocked_compare_store;
+ // InterlockedCompareExchange adds `compare_value` before `value` and
+ // always reports the previous value, so it takes four arguments.
+ const bool IsCompareExchange =
+ BuiltinID == Builtin::BI__builtin_hlsl_interlocked_compare_exchange;
// InterlockedExchange always reports the previous value, so it requires
// `original_value` instead of accepting it as an optional argument.
- if (IsCompareStore ||
- BuiltinID == Builtin::BI__builtin_hlsl_interlocked_exchange) {
+ if (IsCompareExchange) {
+ if (SemaRef.checkArgCount(TheCall, 4))
+ return true;
+ } else if (IsCompareStore ||
+ BuiltinID == Builtin::BI__builtin_hlsl_interlocked_exchange) {
if (SemaRef.checkArgCount(TheCall, 3))
return true;
} else {
@@ -4783,17 +4791,21 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
{LangAS::hlsl_groupshared, LangAS::hlsl_device}))
return true;
- if (CheckArgTypeMatches(&SemaRef, TheCall->getArg(1), DestTy))
- return true;
-
- if (TheCall->getNumArgs() == 3) {
- if (CheckArgTypeMatches(&SemaRef, TheCall->getArg(2), DestTy))
- return true;
- // Only the read-modify-write operations write the previous value back
- // through the third argument. For compare-store it is the new value.
- if (!IsCompareStore && CheckModifiableLValue(&SemaRef, TheCall, 2))
+ // Every argument after `dest` has the destination's type.
+ for (unsigned I = 1, E = TheCall->getNumArgs(); I != E; ++I)
+ if (CheckArgTypeMatches(&SemaRef, TheCall->getArg(I), DestTy))
return true;
- }
+
+ // Operations that report the previous value write it back through their
+ // last argument. Compare-store reports nothing, so its last argument is
+ // the new value, and the two-argument read-modify-write forms have no
+ // such argument at all.
+ const unsigned NumArgs = TheCall->getNumArgs();
+ const bool HasOriginalValue =
+ !IsCompareStore && NumArgs == (IsCompareExchange ? 4u : 3u);
+ if (HasOriginalValue &&
+ CheckModifiableLValue(&SemaRef, TheCall, NumArgs - 1))
+ return true;
TheCall->setType(SemaRef.Context.VoidTy);
break;
diff --git a/clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl b/clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl
new file mode 100644
index 0000000000000..2ecb60edbcf4a
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl
@@ -0,0 +1,70 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \
+// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK
+
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \
+// RUN: spirv-pc-vulkan-library %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,SPVCHECK
+
+// Test basic lowering of HLSL InterlockedCompareExchange to `cmpxchg
+// monotonic`. The operation reports the value that was in the destination
+// before the operation, so the first element of the `cmpxchg` result goes to
+// the out parameter.
+
+groupshared int gs_i32;
+groupshared uint gs_u32;
+groupshared int64_t gs_i64;
+groupshared uint64_t gs_u64;
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int
+// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic
+// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic
+// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i32, i1 } [[PAIR]], 0
+// CHECK-NEXT: store i32 [[OLD]], ptr {{.*}}%orig
+export void test_int(int cmp, int v) {
+ int orig;
+ InterlockedCompareExchange(gs_i32, cmp, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint
+// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic
+// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic
+// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i32, i1 } [[PAIR]], 0
+// CHECK-NEXT: store i32 [[OLD]], ptr {{.*}}%orig
+export void test_uint(uint cmp, uint v) {
+ uint orig;
+ InterlockedCompareExchange(gs_u32, cmp, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int64
+// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic
+// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic
+// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i64, i1 } [[PAIR]], 0
+// CHECK-NEXT: store i64 [[OLD]], ptr {{.*}}%orig
+export void test_int64(int64_t cmp, int64_t v) {
+ int64_t orig;
+ InterlockedCompareExchange(gs_i64, cmp, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint64
+// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic
+// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic
+// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i64, i1 } [[PAIR]], 0
+// CHECK-NEXT: store i64 [[OLD]], ptr {{.*}}%orig
+export void test_uint64(uint64_t cmp, uint64_t v) {
+ uint64_t orig;
+ InterlockedCompareExchange(gs_u64, cmp, v, orig);
+}
+
+// A device-address-space destination uses the "device" scope instead.
+RWBuffer<uint> Buf : register(u0);
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_device
+// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic
+// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(11) %{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic
+// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i32, i1 } [[PAIR]], 0
+// CHECK-NEXT: store i32 [[OLD]], ptr {{.*}}%orig
+export void test_device(uint cmp, uint v) {
+ uint orig;
+ InterlockedCompareExchange(Buf[0], cmp, v, orig);
+}
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl
index 3af09218eec62..47cff714a7df7 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl
@@ -41,6 +41,8 @@ RWBuffer<uint> UOut : register(u1);
// DXCHECK: atomicrmw xchg ptr %[[PTR9]], i32 1 syncscope("device") monotonic
// DXCHECK: %[[PTR10:.*]] = call {{.*}} @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t.i32(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
// DXCHECK: cmpxchg ptr %[[PTR10]], i32 1, i32 2 syncscope("device") monotonic monotonic
+// DXCHECK: %[[PTR11:.*]] = call {{.*}} @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t.i32(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
+// DXCHECK: cmpxchg ptr %[[PTR11]], i32 1, i32 2 syncscope("device") monotonic monotonic
// SPVCHECK: %[[PTR1:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}})
// SPVCHECK: atomicrmw add ptr addrspace(11) %[[PTR1]], i32 1 syncscope("device") monotonic
// SPVCHECK: %[[PTR2:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}})
@@ -61,6 +63,8 @@ RWBuffer<uint> UOut : register(u1);
// SPVCHECK: atomicrmw xchg ptr addrspace(11) %[[PTR9]], i32 1 syncscope("device") monotonic
// SPVCHECK: %[[PTR10:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}})
// SPVCHECK: cmpxchg ptr addrspace(11) %[[PTR10]], i32 1, i32 2 syncscope("device") monotonic monotonic
+// SPVCHECK: %[[PTR11:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}})
+// SPVCHECK: cmpxchg ptr addrspace(11) %[[PTR11]], i32 1, i32 2 syncscope("device") monotonic monotonic
[shader("compute")]
[numthreads(1,1,1)]
void main(uint3 id : SV_DispatchThreadID) {
@@ -75,4 +79,5 @@ void main(uint3 id : SV_DispatchThreadID) {
int orig;
InterlockedExchange(Out[id.x], 1, orig);
InterlockedCompareStore(Out[id.x], 1, 2);
+ InterlockedCompareExchange(Out[id.x], 1, 2, orig);
}
diff --git a/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl
new file mode 100644
index 0000000000000..40d391da06cbe
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl
@@ -0,0 +1,44 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \
+// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK
+
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \
+// RUN: spirv-pc-vulkan1.3-library %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,SPVCHECK
+
+// Test that the RWByteAddressBuffer::InterlockedCompareExchange and
+// InterlockedCompareExchange64 member methods lower to `resource_getpointer ->
+// cmpxchg`, for both DXIL and SPIR-V targets. The value that was in the
+// destination before the operation goes to the out parameter.
+
+RWByteAddressBuffer BAB : register(u0);
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic
+// SPVCHECK: %[[HANDLE:.*]] = load target("spirv.VulkanBuffer", [0 x i8], 12, 1), ptr {{.*}}
+// SPVCHECK: %[[PTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i8_12_1t.i32(target("spirv.VulkanBuffer", [0 x i8], 12, 1) %[[HANDLE]], i32 %{{.*}})
+// SPVCHECK: %[[PAIR:.*]] = cmpxchg ptr addrspace(11) %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic
+// CHECK-NEXT: %[[OLD:.*]] = extractvalue { i32, i1 } %[[PAIR]], 0
+// CHECK-NEXT: %[[OUT:.*]] = load ptr{{.*}}, ptr {{.*}}%OriginalValue.addr
+// CHECK-NEXT: store i32 %[[OLD]], ptr{{.*}} %[[OUT]]
+export void test_bab_uint(uint off, uint cmp, uint v) {
+ uint orig;
+ BAB.InterlockedCompareExchange(off, cmp, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic
+// SPVCHECK: %[[HANDLE:.*]] = load target("spirv.VulkanBuffer", [0 x i8], 12, 1), ptr {{.*}}
+// SPVCHECK: %[[PTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i8_12_1t.i32(target("spirv.VulkanBuffer", [0 x i8], 12, 1) %[[HANDLE]], i32 %{{.*}})
+// SPVCHECK: %[[PAIR:.*]] = cmpxchg ptr addrspace(11) %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic
+// CHECK-NEXT: %[[OLD:.*]] = extractvalue { i64, i1 } %[[PAIR]], 0
+// CHECK-NEXT: %[[OUT:.*]] = load ptr{{.*}}, ptr {{.*}}%OriginalValue.addr
+// CHECK-NEXT: store i64 %[[OLD]], ptr{{.*}} %[[OUT]]
+export void test_bab_uint64(uint off, uint64_t cmp, uint64_t v) {
+ uint64_t orig;
+ BAB.InterlockedCompareExchange64(off, cmp, v, orig);
+}
diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl
new file mode 100644
index 0000000000000..e69b12cde0586
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl
@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \
+// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK
+
+// SPIR-V codegen for RasterizerOrderedByteAddressBuffer is not implemented
+// yet (asserts in clang/lib/CodeGen/Targets/SPIR.cpp on
+// `!ResAttrs.IsROV && "Rasterizer order views not implemented for SPIR-V yet"`).
+// Add a `spirv-pc-vulkan1.3-library` RUN line here when SPIR-V ROV support
+// lands.
+
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+
+// CHECK-LABEL: define void @{{.*}}test_rovb_uint
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic
+// DXCHECK-NEXT: %[[OLD:.*]] = extractvalue { i32, i1 } %[[PAIR]], 0
+// DXCHECK-NEXT: %[[OUT:.*]] = load ptr, ptr {{.*}}%OriginalValue.addr
+// DXCHECK-NEXT: store i32 %[[OLD]], ptr %[[OUT]]
+export void test_rovb_uint(uint off, uint cmp, uint v) {
+ uint orig;
+ ROVB.InterlockedCompareExchange(off, cmp, v, orig);
+}
+
+// CHECK-LABEL: define void @{{.*}}test_rovb_uint64
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic
+// DXCHECK-NEXT: %[[OLD:.*]] = extractvalue { i64, i1 } %[[PAIR]], 0
+// DXCHECK-NEXT: %[[OUT:.*]] = load ptr, ptr {{.*}}%OriginalValue.addr
+// DXCHECK-NEXT: store i64 %[[OLD]], ptr %[[OUT]]
+export void test_rovb_uint64(uint off, uint64_t cmp, uint64_t v) {
+ uint64_t orig;
+ ROVB.InterlockedCompareExchange64(off, cmp, v, orig);
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl
new file mode 100644
index 0000000000000..cdb6a8c9969bf
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.5-library %s -fsyntax-only -verify \
+// RUN: -verify-ignore-unexpected=warning
+
+RWByteAddressBuffer BAB : register(u0);
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+
+void sm65_no_bab_compare_exchange64(uint off, uint64_t cmp, uint64_t v) {
+ uint64_t orig;
+ BAB.InterlockedCompareExchange64(off, cmp, v, orig);
+ // expected-error at -1 {{no member named 'InterlockedCompareExchange64' in 'hlsl::RWByteAddressBuffer'}}
+}
+
+void sm65_no_rovb_compare_exchange64(uint off, uint64_t cmp, uint64_t v) {
+ uint64_t orig;
+ ROVB.InterlockedCompareExchange64(off, cmp, v, orig);
+ // expected-error at -1 {{no member named 'InterlockedCompareExchange64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}}
+}
+
+void sm65_bab_compare_exchange32_ok(uint off, uint cmp, uint v) {
+ uint orig;
+ BAB.InterlockedCompareExchange(off, cmp, v, orig);
+}
+
+groupshared int64_t gs_i64;
+void sm65_direct_builtin(int64_t cmp, int64_t v) {
+ int64_t orig;
+ __builtin_hlsl_interlocked_compare_exchange(gs_i64, cmp, v, orig);
+ // expected-error at -1 {{'__builtin_hlsl_interlocked_compare_exchange' requires shader model 6.6 or newer}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl
new file mode 100644
index 0000000000000..468131f43d01e
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl
@@ -0,0 +1,114 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only \
+// RUN: -disable-llvm-passes -verify
+
+// InterlockedCompareExchange is provided as a set of address-space-qualified
+// overloads (groupshared/device, {int,uint,int64_t,uint64_t}). It reports the
+// value that was in the destination, so it has a single 4-argument form with a
+// trailing out parameter. Float compare-exchange is a separate intrinsic, so
+// float and double have no overload.
+
+groupshared int gs_i32;
+groupshared float gs_f32;
+struct S { int x; };
+groupshared S gs_s;
+
+void too_few(int cmp, int v) {
+ InterlockedCompareExchange(gs_i32, cmp, v); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}}
+ // expected-note@*:* 8 {{candidate function}}
+}
+
+void too_many(int cmp, int v, int extra) {
+ int orig;
+ InterlockedCompareExchange(gs_i32, cmp, v, orig, extra); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}}
+ // expected-note@*:* 8 {{candidate function}}
+}
+
+void local_dest(int cmp, int v) {
+ int dest;
+ int orig;
+ InterlockedCompareExchange(dest, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}}
+ // expected-note@*:* 8 {{candidate function}}
+}
+
+void float_dest(float cmp, float v) {
+ float orig;
+ InterlockedCompareExchange(gs_f32, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}}
+ // expected-note@*:* 8 {{candidate function}}
+}
+
+void struct_dest(int cmp, int v) {
+ S orig;
+ InterlockedCompareExchange(gs_s, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}}
+ // expected-note@*:* 8 {{candidate function}}
+}
+
+// The out parameter is a reference, so it cannot bind across types.
+void mismatched_orig_type(int cmp, int v) {
+ float orig;
+ InterlockedCompareExchange(gs_i32, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}}
+ // expected-note@*:* 8 {{candidate function}}
+}
+
+void direct_too_few(int cmp, int v) {
+ __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, v);
+ // expected-error at -1 {{too few arguments to function call, expected 4, have 3}}
+}
+
+void direct_too_many(int cmp, int v, int extra) {
+ int orig;
+ __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, v, orig, extra);
+ // expected-error at -1 {{too many arguments to function call, expected 4, have 5}}
+}
+
+void direct_non_integer_dest() {
+ S local_s;
+ int orig;
+ __builtin_hlsl_interlocked_compare_exchange(local_s, 1, 2, orig);
+ // expected-error at -1 {{1st argument must be a scalar integer type (was 'S')}}
+}
+
+void direct_float_dest(float cmp, float v) {
+ float orig;
+ __builtin_hlsl_interlocked_compare_exchange(gs_f32, cmp, v, orig);
+ // expected-error at -1 {{1st argument must be a scalar integer type (was 'float')}}
+}
+
+void direct_nonlvalue_dest(int cmp, int v) {
+ int orig;
+ __builtin_hlsl_interlocked_compare_exchange(1, cmp, v, orig);
+ // expected-error at -1 {{cannot bind non-lvalue argument '1' to out parameter}}
+}
+
+// The last argument is an out parameter, so an rvalue is rejected there.
+void direct_nonlvalue_original_value(int cmp, int v) {
+ __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, v, 0);
+ // expected-error at -1 {{cannot bind non-lvalue argument '0' to out parameter}}
+}
+
+void direct_mismatched_compare() {
+ uint cmp = 1;
+ int orig;
+ __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, 2, orig);
+ // expected-error at -1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}}
+}
+
+void direct_mismatched_value() {
+ uint v = 1;
+ int orig;
+ __builtin_hlsl_interlocked_compare_exchange(gs_i32, 1, v, orig);
+ // expected-error at -1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}}
+}
+
+void direct_mismatched_original_value() {
+ uint orig;
+ __builtin_hlsl_interlocked_compare_exchange(gs_i32, 1, 2, orig);
+ // expected-error at -1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}}
+}
+
+void direct_default_as_dest(int cmp, int v) {
+ int local;
+ int orig;
+ __builtin_hlsl_interlocked_compare_exchange(local, cmp, v, orig);
+ // expected-error at -1 {{1st argument to atomic builtin must reference groupshared or device memory (was 'int')}}
+}
More information about the llvm-branch-commits
mailing list