[clang] [llvm] Add InterlockedAdd resource methods (PR #208128)
Joshua Batista via cfe-commits
cfe-commits at lists.llvm.org
Fri Jul 17 14:52:14 PDT 2026
https://github.com/bob80905 updated https://github.com/llvm/llvm-project/pull/208128
>From 997234df5400207a5de0c750ed0ec12b70475e0a Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Tue, 7 Jul 2026 18:00:56 -0700
Subject: [PATCH 1/7] first attempt, add interlockedadd resource methods
---
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 68 +++++++++
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h | 4 +
clang/lib/Sema/HLSLExternalSemaSource.cpp | 2 +
clang/lib/Sema/SemaHLSL.cpp | 23 +++
.../ByteAddressBuffer-InterlockedAdd.hlsl | 80 ++++++++++
...teAddressBuffer-InterlockedAdd-errors.hlsl | 63 ++++++++
llvm/include/llvm/IR/IntrinsicsDirectX.td | 8 +
llvm/lib/Target/DirectX/DXIL.td | 11 ++
llvm/lib/Target/DirectX/DXILOpLowering.cpp | 30 ++++
.../lib/Target/DirectX/DXILResourceAccess.cpp | 141 ++++++++++++++++++
.../DirectX/ResourceAtomicBinOp-i64-sm65.ll | 16 ++
.../CodeGen/DirectX/ResourceAtomicBinOp.ll | 59 ++++++++
12 files changed, 505 insertions(+)
create mode 100644 clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll
create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
index f8018729b4644..245487b71875f 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
@@ -1635,6 +1635,38 @@ BuiltinTypeDeclBuilder::addByteAddressBufferStoreMethods() {
return *this;
}
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() {
+ assert(!Record->isCompleteDefinition() && "record is already complete");
+ ASTContext &AST = SemaRef.getASTContext();
+
+ // Each entry declares two overloads (with and without an out original-value
+ // parameter). Adding a new atomic here only requires a new line — the shared
+ // helper takes care of the composition.
+ addByteAddressBufferInterlockedMethod("InterlockedAdd", AST.UnsignedIntTy,
+ "__builtin_hlsl_interlocked_add");
+
+ // 64-bit typed atomics on UAVs require SM 6.6 (DXIL 1.6 introduces the
+ // int64 overload of the atomicBinOp op). Skip synthesizing the *64 methods
+ // on older DXIL targets so callers get "no matching member function" from
+ // overload resolution — this matches DXC and mirrors how other HLSL SM-gated
+ // features are handled (see hlsl_intrinsics.h `_HLSL_AVAILABILITY`
+ // annotations). Non-DXIL targets (e.g., SPIR-V) always get the method: their
+ // 64-bit atomic support is gated by device extensions, not shader model.
+ const llvm::Triple &TT = AST.getTargetInfo().getTriple();
+ bool DXILNeedsSM66 =
+ TT.getArch() == llvm::Triple::dxil &&
+ AST.getTargetInfo().getPlatformMinVersion() < VersionTuple(6, 6);
+ if (!DXILNeedsSM66) {
+ // HLSL's uint64_t is `unsigned long`.
+ addByteAddressBufferInterlockedMethod("InterlockedAdd64",
+ AST.UnsignedLongTy,
+ "__builtin_hlsl_interlocked_add");
+ }
+
+ return *this;
+}
+
BuiltinTypeDeclBuilder &
BuiltinTypeDeclBuilder::addSampleMethods(ResourceDimension Dim, bool IsArray) {
assert(!Record->isCompleteDefinition() && "record is already complete");
@@ -2356,6 +2388,42 @@ BuiltinTypeDeclBuilder::addStoreFunction(DeclarationName &Name, bool IsConst,
.finalize();
}
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethod(
+ StringRef MethodName, QualType ValueTy, StringRef BuiltinName) {
+ assert(!Record->isCompleteDefinition() && "record is already complete");
+ ASTContext &AST = SemaRef.getASTContext();
+ using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+
+ // Interlocked atomics operate on a typed slot in the buffer. Compose
+ // `resource_getpointer_typed` with the scalar `__builtin_hlsl_interlocked_*`
+ // builtin so backend lowering (DXIL and SPIR-V) can pattern-match a
+ // resource-pointer atomicrmw.
+ QualType AddrSpaceElemTy =
+ AST.getAddrSpaceQualType(ValueTy, LangAS::hlsl_device);
+ QualType ElemPtrTy = AST.getPointerType(AddrSpaceElemTy);
+
+ auto BuildOverload = [&](bool WithOriginalValue) {
+ BuiltinTypeMethodBuilder MMB(*this, MethodName, AST.VoidTy);
+ MMB.addParam("Offset", AST.UnsignedIntTy).addParam("Value", ValueTy);
+ if (WithOriginalValue)
+ MMB.addParam("OriginalValue", ValueTy,
+ HLSLParamModifierAttr::Keyword_out);
+ MMB.callBuiltin("__builtin_hlsl_resource_getpointer_typed", ElemPtrTy,
+ PH::Handle, PH::_0, ValueTy)
+ .dereference(PH::LastStmt);
+ if (WithOriginalValue)
+ MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2);
+ else
+ MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1);
+ MMB.finalize();
+ };
+
+ BuildOverload(/*WithOriginalValue=*/false);
+ BuildOverload(/*WithOriginalValue=*/true);
+ return *this;
+}
+
BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() {
using PH = BuiltinTypeMethodBuilder::PlaceHolder;
ASTContext &AST = SemaRef.getASTContext();
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
index 09cf1fceca116..e809ef264198c 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
@@ -106,6 +106,7 @@ class BuiltinTypeDeclBuilder {
bool IsArray = false);
BuiltinTypeDeclBuilder &addByteAddressBufferLoadMethods();
BuiltinTypeDeclBuilder &addByteAddressBufferStoreMethods();
+ BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedMethods();
BuiltinTypeDeclBuilder &addSampleMethods(ResourceDimension Dim,
bool IsArray = false);
BuiltinTypeDeclBuilder &addSampleBiasMethods(ResourceDimension Dim,
@@ -134,6 +135,9 @@ class BuiltinTypeDeclBuilder {
QualType ReturnTy = QualType());
BuiltinTypeDeclBuilder &addStoreFunction(DeclarationName &Name, bool IsConst,
QualType ValueType);
+ BuiltinTypeDeclBuilder &
+ addByteAddressBufferInterlockedMethod(StringRef MethodName, QualType ValueTy,
+ StringRef BuiltinName);
BuiltinTypeDeclBuilder &addAppendMethod();
BuiltinTypeDeclBuilder &addConsumeMethod();
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index 7578f20a27f18..605d630489275 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -661,6 +661,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
/*RawBuffer=*/true, /*HasCounter=*/false)
.addByteAddressBufferLoadMethods()
.addByteAddressBufferStoreMethods()
+ .addByteAddressBufferInterlockedMethods()
.addGetDimensionsMethodForBuffer()
.completeDefinition();
});
@@ -670,6 +671,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
onCompletion(Decl, [this](CXXRecordDecl *Decl) {
setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, /*IsROV=*/true,
/*RawBuffer=*/true, /*HasCounter=*/false)
+ .addByteAddressBufferInterlockedMethods()
.addGetDimensionsMethodForBuffer()
.completeDefinition();
});
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index c333f0dd4c872..4cedfe893d91c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -4590,6 +4590,29 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
return true;
}
+ // 64-bit interlocked ops require SM 6.6 on DXIL — the DXIL 1.6 int64
+ // overloads of atomicBinOp/cmpXchg are what enable them. The synthesized
+ // wrapper methods (e.g. RWByteAddressBuffer::InterlockedAdd64) that call
+ // this builtin are themselves only declared when the target supports it
+ // (see HLSLBuiltinTypeDeclBuilder), so pre-SM6.6 usage is caught by
+ // overload resolution. This defensive check catches direct
+ // `__builtin_hlsl_interlocked_add` calls from HLSL code with a 64-bit
+ // dest on pre-SM6.6 DXIL targets. Skip synthetic invocations (invalid
+ // source location) built while composing wrapper method bodies.
+ const TargetInfo &TI = SemaRef.Context.getTargetInfo();
+ if (TheCall->getBeginLoc().isValid() &&
+ TI.getTriple().getArch() == llvm::Triple::dxil &&
+ SemaRef.Context.getTypeSize(DestTy) == 64 &&
+ TI.getPlatformMinVersion() < VersionTuple(6, 6)) {
+ llvm::StringRef PlatformName(
+ AvailabilityAttr::getPrettyPlatformName(TI.getPlatformName()));
+ SemaRef.Diag(TheCall->getBeginLoc(), diag::warn_hlsl_availability)
+ << TheCall->getDirectCallee() << PlatformName
+ << VersionTuple(6, 6).getAsString() << /*UseEnvironment=*/false
+ << /*EnvName=*/"";
+ return true;
+ }
+
if (CheckModifiableLValue(&SemaRef, TheCall, 0))
return true;
diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
new file mode 100644
index 0000000000000..ae6179dc4bc99
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
@@ -0,0 +1,80 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \
+// RUN: dxil-pc-shadermodel6.6-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK
+
+// Test that the RWByteAddressBuffer::InterlockedAdd and
+// RasterizerOrderedByteAddressBuffer::InterlockedAdd member methods lower to
+// `dx.resource.getpointer.typed -> dx.interlocked.add`, and that the
+// 3-argument overload stores the returned original value through the out
+// parameter.
+
+RWByteAddressBuffer BAB : register(u0);
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+export void test_bab_int_2arg(uint off, int v) {
+ BAB.InterlockedAdd(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: store i32 %[[R]], ptr {{.*}}
+export void test_bab_uint_3arg(uint off, uint v, out uint orig) {
+ BAB.InterlockedAdd(off, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+export void test_rovb_int_2arg(uint off, int v) {
+ ROVB.InterlockedAdd(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: store i32 %[[R]], ptr {{.*}}
+export void test_rovb_uint_3arg(uint off, uint v, out uint orig) {
+ ROVB.InterlockedAdd(off, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int64_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+export void test_bab_int64_2arg(uint off, int64_t v) {
+ BAB.InterlockedAdd64(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: store i64 %[[R]], ptr {{.*}}
+export void test_bab_uint64_3arg(uint off, uint64_t v, out uint64_t orig) {
+ BAB.InterlockedAdd64(off, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int64_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+export void test_rovb_int64_2arg(uint off, int64_t v) {
+ ROVB.InterlockedAdd64(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint64_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: store i64 %[[R]], ptr {{.*}}
+export void test_rovb_uint64_3arg(uint off, uint64_t v, out uint64_t orig) {
+ ROVB.InterlockedAdd64(off, v, orig);
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
new file mode 100644
index 0000000000000..13fc70d700421
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \
+// RUN: -verify-ignore-unexpected=note,warning
+
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.5-compute -DTEST_SM65 %s -fsyntax-only \
+// RUN: -verify -verify-ignore-unexpected=note,warning
+
+RWByteAddressBuffer BAB : register(u0);
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+
+struct S { int x; };
+
+#ifdef TEST_SM65
+
+// InterlockedAdd64 is only synthesized on DXIL when the shader model is at
+// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so
+// the reference must fail with "no member named".
+void sm65_no_bab_add64(uint off, int64_t v) {
+ BAB.InterlockedAdd64(off, v);
+ // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}}
+}
+
+void sm65_no_rovb_add64(uint off, int64_t v) {
+ ROVB.InterlockedAdd64(off, v);
+ // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}}
+}
+
+// 32-bit InterlockedAdd is always available.
+void sm65_bab_add32_ok(uint off, int v) {
+ BAB.InterlockedAdd(off, v);
+}
+
+#else
+
+void too_few(uint off) {
+ BAB.InterlockedAdd(off);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+void too_many(uint off, int v, int extra) {
+ int orig;
+ BAB.InterlockedAdd(off, v, orig, extra);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+void struct_value(uint off, S v) {
+ BAB.InterlockedAdd(off, v);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+// Same shape of errors on RasterizerOrderedByteAddressBuffer.
+void rovb_too_few(uint off) {
+ ROVB.InterlockedAdd(off);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+void rovb_struct_value(uint off, S v) {
+ ROVB.InterlockedAdd(off, v);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 4dd86270f0d01..f1b3845ea1efa 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -61,6 +61,14 @@ def int_dx_resource_store_rawbuffer
: DefaultAttrsIntrinsic<
[], [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, llvm_any_ty],
[IntrWriteMem]>;
+// Resource atomic binary op: performs an atomic read-modify-write on a UAV
+// resource element and returns the original value. The i32 operation code
+// matches DXIL's AtomicBinOpCode enum.
+def int_dx_resource_atomicbinop
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [llvm_any_ty, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, LLVMMatchType<0>],
+ [IntrArgMemOnly]>;
// dx.resource.load.cbufferrow encodes the number of elements returned in the
// function name. The total size of the return should always be 128 bits.
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index a268276b07655..7c57c8a7f4aa1 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -1006,6 +1006,17 @@ def BufferStore : DXILOp<69, bufferStore> {
let stages = [Stages<DXIL1_0, [all_stages]>];
}
+def AtomicBinOp : DXILOp<78, atomicBinOp> {
+ let Doc = "performs an atomic read-modify-write on a UAV resource "
+ "element, returning the original value";
+ // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue
+ let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy];
+ let result = OverloadTy;
+ let overloads = [Overloads<DXIL1_0, [Int32Ty]>,
+ Overloads<DXIL1_6, [Int32Ty, Int64Ty]>];
+ let stages = [Stages<DXIL1_0, [all_stages]>];
+}
+
def UpdateCounter : DXILOp<70, bufferUpdateCounter> {
let Doc = "increments/decrements a buffer counter";
let arguments = [HandleTy, Int8Ty];
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 93d5a08a6e0a2..64e0de8e55bdd 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -849,6 +849,33 @@ class OpLowerer {
});
}
+ [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) {
+ IRBuilder<> &IRB = OpBuilder.getIRB();
+
+ return replaceFunction(F, [&](CallInst *CI) -> Error {
+ IRB.SetInsertPoint(CI);
+ Value *Handle =
+ createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());
+ Value *Index = CI->getArgOperand(1);
+ Value *Offset = CI->getArgOperand(2);
+ Value *BinOp = CI->getArgOperand(3);
+ Value *NewValue = CI->getArgOperand(4);
+
+ std::array<Value *, 6> Args{Handle, BinOp, Index,
+ Offset, IRB.getInt32(0), NewValue};
+
+ Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
+ OpCode::AtomicBinOp, Args, CI->getName(), CI->getType());
+
+ if (Error E = OpCall.takeError())
+ return E;
+
+ CI->replaceAllUsesWith(*OpCall);
+ CI->eraseFromParent();
+ return Error::success();
+ });
+ }
+
[[nodiscard]] bool lowerGetDimensionsX(Function &F) {
IRBuilder<> &IRB = OpBuilder.getIRB();
Type *Int32Ty = IRB.getInt32Ty();
@@ -1210,6 +1237,9 @@ class OpLowerer {
case Intrinsic::dx_resource_updatecounter:
HasErrors |= lowerUpdateCounter(F);
break;
+ case Intrinsic::dx_resource_atomicbinop:
+ HasErrors |= lowerResourceAtomicBinOp(F);
+ break;
case Intrinsic::dx_resource_getdimensions_x:
HasErrors |= lowerGetDimensionsX(F);
break;
diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
index 25d860e615c17..5f5c4660e744b 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
@@ -26,6 +26,7 @@
#include "llvm/InitializePasses.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
+#include <optional>
#define DEBUG_TYPE "dxil-resource-access"
@@ -230,6 +231,114 @@ static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,
llvm_unreachable("Unhandled case in switch");
}
+static std::optional<unsigned> getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) {
+ switch (BinOp) {
+ case AtomicRMWInst::Add:
+ return 0;
+ case AtomicRMWInst::And:
+ return 1;
+ case AtomicRMWInst::Or:
+ return 2;
+ case AtomicRMWInst::Xor:
+ return 3;
+ case AtomicRMWInst::Min:
+ return 4;
+ case AtomicRMWInst::Max:
+ return 5;
+ case AtomicRMWInst::UMin:
+ return 6;
+ case AtomicRMWInst::UMax:
+ return 7;
+ case AtomicRMWInst::Xchg:
+ return 8;
+ case AtomicRMWInst::Sub:
+ case AtomicRMWInst::Nand:
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ case AtomicRMWInst::FMax:
+ case AtomicRMWInst::FMin:
+ case AtomicRMWInst::FMaximum:
+ case AtomicRMWInst::FMinimum:
+ case AtomicRMWInst::FMaximumNum:
+ case AtomicRMWInst::FMinimumNum:
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
+ case AtomicRMWInst::BAD_BINOP:
+ return std::nullopt;
+ }
+ llvm_unreachable("Unhandled atomicrmw operation");
+}
+
+static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
+ dxil::ResourceTypeInfo &RTI) {
+ std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation());
+ if (!BinOpCode) {
+ reportFatalUsageError("DXIL resource atomicrmw operation not implemented");
+ return;
+ }
+
+ const DataLayout &DL = AI->getDataLayout();
+ IRBuilder<> Builder(AI);
+ Value *Index = II->getOperand(1);
+
+ // The offset for the rawbuffer load/store/atomic ops is always in bytes.
+ uint64_t AccessSize = 1;
+ Value *Offset =
+ traverseGEPOffsets(DL, Builder, AI->getPointerOperand(), AccessSize);
+
+ // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
+ // entirely into the index.
+ if (!RTI.isStruct()) {
+ auto *ConstantOffset = dyn_cast<ConstantInt>(Offset);
+ if (!ConstantOffset || !ConstantOffset->isZero())
+ Index = Builder.CreateAdd(Index, Offset);
+ Offset = llvm::PoisonValue::get(Builder.getInt32Ty());
+ }
+
+ auto *BinOp = Builder.getInt32(*BinOpCode);
+ Value *V = Builder.CreateIntrinsic(
+ AI->getType(), Intrinsic::dx_resource_atomicbinop,
+ {II->getOperand(0), Index, Offset, BinOp, AI->getValOperand()});
+ AI->replaceAllUsesWith(V);
+}
+
+static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
+ dxil::ResourceTypeInfo &RTI) {
+ switch (RTI.getResourceKind()) {
+ case dxil::ResourceKind::TypedBuffer:
+ case dxil::ResourceKind::RawBuffer:
+ case dxil::ResourceKind::StructuredBuffer:
+ return createAtomicBinOp(II, AI, RTI);
+ case dxil::ResourceKind::Texture1D:
+ case dxil::ResourceKind::Texture2D:
+ case dxil::ResourceKind::Texture2DMS:
+ case dxil::ResourceKind::Texture3D:
+ case dxil::ResourceKind::TextureCube:
+ case dxil::ResourceKind::Texture1DArray:
+ case dxil::ResourceKind::Texture2DArray:
+ case dxil::ResourceKind::Texture2DMSArray:
+ case dxil::ResourceKind::TextureCubeArray:
+ case dxil::ResourceKind::FeedbackTexture2D:
+ case dxil::ResourceKind::FeedbackTexture2DArray:
+ reportFatalUsageError(
+ "DXIL atomicrmw not implemented for texture resources");
+ return;
+ case dxil::ResourceKind::CBuffer:
+ case dxil::ResourceKind::Sampler:
+ case dxil::ResourceKind::TBuffer:
+ reportFatalUsageError(
+ "DXIL atomicrmw not implemented for this resource type");
+ return;
+ case dxil::ResourceKind::RTAccelerationStructure:
+ case dxil::ResourceKind::Invalid:
+ case dxil::ResourceKind::NumEntries:
+ llvm_unreachable("Invalid resource kind for atomicrmw");
+ }
+ llvm_unreachable("Unhandled case in switch");
+}
+
static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,
dxil::ResourceTypeInfo &RTI) {
const DataLayout &DL = LI->getDataLayout();
@@ -550,6 +659,8 @@ static Instruction *getStoreLoadPointerOperand(Instruction *AI) {
return dyn_cast<Instruction>(LI->getPointerOperand());
if (auto *SI = dyn_cast<StoreInst>(AI))
return dyn_cast<Instruction>(SI->getPointerOperand());
+ if (auto *RMWI = dyn_cast<AtomicRMWInst>(AI))
+ return dyn_cast<Instruction>(RMWI->getPointerOperand());
return nullptr;
}
@@ -786,6 +897,36 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
} else if (auto *LI = dyn_cast<LoadInst>(U)) {
createLoadIntrinsic(II, LI, RTI);
DeadInsts.push_back(LI);
+ } else if (auto *AI = dyn_cast<AtomicRMWInst>(U)) {
+ createAtomicBinOpIntrinsic(II, AI, RTI);
+ DeadInsts.push_back(AI);
+ } else if (auto *CI = dyn_cast<CallInst>(U)) {
+ // `dx.interlocked.*` intrinsics wrap an atomicrmw and are expanded to
+ // one by DXILIntrinsicExpansion — but that pass runs after this one, so
+ // when the source of the pointer is a resource we must expand them here
+ // (and immediately process the resulting atomicrmw) instead of letting
+ // the pointer escape.
+ auto *IntrinCall = dyn_cast<IntrinsicInst>(CI);
+ std::optional<AtomicRMWInst::BinOp> Op;
+ if (IntrinCall) {
+ switch (IntrinCall->getIntrinsicID()) {
+ case Intrinsic::dx_interlocked_add:
+ Op = AtomicRMWInst::Add;
+ break;
+ default:
+ break;
+ }
+ }
+ if (!Op)
+ llvm_unreachable("Unhandled instruction - pointer escaped?");
+ IRBuilder<> Builder(IntrinCall);
+ auto *AI = Builder.CreateAtomicRMW(
+ *Op, IntrinCall->getArgOperand(0), IntrinCall->getArgOperand(1),
+ MaybeAlign(), AtomicOrdering::Monotonic);
+ IntrinCall->replaceAllUsesWith(AI);
+ createAtomicBinOpIntrinsic(II, AI, RTI);
+ DeadInsts.push_back(AI);
+ DeadInsts.push_back(IntrinCall);
} else
llvm_unreachable("Unhandled instruction - pointer escaped?");
}
diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll
new file mode 100644
index 0000000000000..6f5a40e0b6c2a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll
@@ -0,0 +1,16 @@
+; RUN: not opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.5-compute %s 2>&1 | FileCheck %s
+
+; Verify resource i64 atomicrmw rejects shader models before SM 6.6, where
+; dx.op.atomicBinOp gained i64 overload support.
+
+target triple = "dxil-pc-shadermodel6.5-compute"
+
+define i64 @atomic_i64(i32 %index, i64 %value) {
+ %buffer = call target("dx.RawBuffer", i64, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index)
+ ; CHECK: Cannot create AtomicBinOp operation: Invalid overload type
+ %old = atomicrmw add ptr %ptr, i64 %value monotonic
+ ret i64 %old
+}
diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll
new file mode 100644
index 0000000000000..bc852c8a2c81f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -dxil-resource-access -dxil-op-lower %s | FileCheck %s --check-prefixes=CHECK,I32
+; RUN: opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s --check-prefixes=CHECK,I32,I64
+
+; Verify atomicrmw through a dx.resource.getpointer is lowered to
+; dx.op.atomicBinOp for UAV resources.
+
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+; CHECK-LABEL: define i32 @atomic_i32(
+define i32 @atomic_i32(i32 %index, i32 %value) {
+ %buffer = call target("dx.RawBuffer", i32, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i32, 1, 0, 0) %buffer, i32 %index)
+
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i32 %value)
+ %add = atomicrmw add ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 1, i32 %index, i32 0, i32 0, i32 %value)
+ %and = atomicrmw and ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 2, i32 %index, i32 0, i32 0, i32 %value)
+ %or = atomicrmw or ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 3, i32 %index, i32 0, i32 0, i32 %value)
+ %xor = atomicrmw xor ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 4, i32 %index, i32 0, i32 0, i32 %value)
+ %min = atomicrmw min ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 5, i32 %index, i32 0, i32 0, i32 %value)
+ %max = atomicrmw max ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 6, i32 %index, i32 0, i32 0, i32 %value)
+ %umin = atomicrmw umin ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 7, i32 %index, i32 0, i32 0, i32 %value)
+ %umax = atomicrmw umax ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 8, i32 %index, i32 0, i32 0, i32 %value)
+ %xchg = atomicrmw xchg ptr %ptr, i32 %value monotonic
+ ret i32 %xchg
+}
+
+; CHECK-LABEL: define i32 @atomic_i32_byteaddress(
+define i32 @atomic_i32_byteaddress(i32 %offset, i32 %value) {
+ %buffer = call target("dx.RawBuffer", i8, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset)
+
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %offset, i32 poison, i32 0, i32 %value)
+ %old = atomicrmw add ptr %ptr, i32 %value monotonic
+ ret i32 %old
+}
+
+; CHECK-LABEL: define i64 @atomic_i64(
+define i64 @atomic_i64(i32 %index, i64 %value) {
+ %buffer = call target("dx.RawBuffer", i64, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 2, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index)
+
+ ; I64: call i64 @dx.op.atomicBinOp.i64(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i64 %value)
+ %old = atomicrmw add ptr %ptr, i64 %value monotonic
+ ret i64 %old
+}
>From b34ffa6e2dc6b1354ddfab9719f86d049a628660 Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Wed, 8 Jul 2026 16:16:27 -0700
Subject: [PATCH 2/7] self review: Move op def to appropriate spot
---
llvm/lib/Target/DirectX/DXIL.td | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 7c57c8a7f4aa1..3d978c207f104 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -1006,17 +1006,6 @@ def BufferStore : DXILOp<69, bufferStore> {
let stages = [Stages<DXIL1_0, [all_stages]>];
}
-def AtomicBinOp : DXILOp<78, atomicBinOp> {
- let Doc = "performs an atomic read-modify-write on a UAV resource "
- "element, returning the original value";
- // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue
- let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy];
- let result = OverloadTy;
- let overloads = [Overloads<DXIL1_0, [Int32Ty]>,
- Overloads<DXIL1_6, [Int32Ty, Int64Ty]>];
- let stages = [Stages<DXIL1_0, [all_stages]>];
-}
-
def UpdateCounter : DXILOp<70, bufferUpdateCounter> {
let Doc = "increments/decrements a buffer counter";
let arguments = [HandleTy, Int8Ty];
@@ -1041,6 +1030,17 @@ def GetDimensions : DXILOp<72, getDimensions> {
let stages = [Stages<DXIL1_0, [all_stages]>];
}
+def AtomicBinOp : DXILOp<78, atomicBinOp> {
+ let Doc = "performs an atomic read-modify-write on a UAV resource "
+ "element, returning the original value";
+ // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue
+ let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy];
+ let result = OverloadTy;
+ let overloads = [Overloads<DXIL1_0, [Int32Ty]>,
+ Overloads<DXIL1_6, [Int32Ty, Int64Ty]>];
+ let stages = [Stages<DXIL1_0, [all_stages]>];
+}
+
def Barrier : DXILOp<80, barrier> {
let Doc = "inserts a memory barrier in the shader";
let intrinsics = [
>From d42ceac6a97b78e22a10b5a74a9ee8ad7ab5cadb Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Wed, 15 Jul 2026 16:11:19 -0700
Subject: [PATCH 3/7] address Farzon
---
.../clang/Basic/DiagnosticSemaKinds.td | 3 +
clang/lib/CodeGen/CGHLSLBuiltins.cpp | 28 ++++++-
clang/lib/CodeGen/CGHLSLRuntime.h | 1 -
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 21 ++---
clang/lib/Sema/SemaHLSL.cpp | 21 ++---
.../ByteAddressBuffer-InterlockedAdd.hlsl | 32 +++----
.../CodeGenHLSL/builtins/InterlockedAdd.hlsl | 24 +++---
...teAddressBuffer-InterlockedAdd-errors.hlsl | 28 -------
...ressBuffer-InterlockedAdd-sm65-errors.hlsl | 32 +++++++
llvm/include/llvm/IR/IntrinsicsDirectX.td | 13 ---
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 4 -
.../Target/DirectX/DXILIntrinsicExpansion.cpp | 4 -
llvm/lib/Target/DirectX/DXILOpLowering.cpp | 43 +++-------
.../lib/Target/DirectX/DXILResourceAccess.cpp | 83 ++++++++++---------
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 2 -
llvm/test/CodeGen/DirectX/InterlockedAdd.ll | 52 ------------
.../SPIRV/hlsl-intrinsics/InterlockedAdd.ll | 36 --------
.../hlsl-intrinsics/InterlockedAdd_spv_i64.ll | 37 ---------
18 files changed, 159 insertions(+), 305 deletions(-)
create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl
delete mode 100644 llvm/test/CodeGen/DirectX/InterlockedAdd.ll
delete mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll
delete mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index f14288dd2967d..fae7a22859425 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -13788,6 +13788,9 @@ def err_hlsl_atomic_arg_addr_space : Error<
"%ordinal0 argument to atomic builtin must reference groupshared or device "
"memory (was %1)">;
+def err_hlsl_builtin_requires_sm : Error<
+ "%0 requires shader model %1 or newer">;
+
def err_hlsl_export_not_on_function : Error<
"export declaration can only be used on functions">;
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index 1bda113143e07..ef8be15fdfb40 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -310,6 +310,27 @@ static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
}
+static Value *handleInterlockedAdd(CodeGenFunction &CGF, const CallExpr *E) {
+ // Emit `atomicrmw add` directly — no intermediate `*.interlocked.add`
+ // intrinsic needed on either DXIL or SPIR-V.
+ LValue DestLV = CGF.EmitLValue(E->getArg(0));
+ Address DestAddr = DestLV.getAddress();
+ Value *Val = CGF.EmitScalarExpr(E->getArg(1));
+ assert(E->getArg(1)->getType()->isIntegerType() &&
+ "Intrinsic InterlockedAdd value operand must be an integer");
+
+ llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW(
+ llvm::AtomicRMWInst::Add, DestAddr, Val, llvm::AtomicOrdering::Monotonic);
+
+ // The 3-arg overload writes the old value (the RMW's return value) into
+ // the `original_value` reference parameter.
+ if (E->getNumArgs() == 3) {
+ LValue OrigLV = CGF.EmitLValue(E->getArg(2));
+ CGF.EmitStoreThroughLValue(RValue::get(Call), OrigLV);
+ }
+ return Call;
+}
+
static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
Intrinsic::ID ID, const Twine &Name) {
// HLSL signatures (synthesized as overloads in HLSLExternalSemaSource):
@@ -1457,9 +1478,10 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
"hlsl.wave.active.bit.and");
}
case Builtin::BI__builtin_hlsl_interlocked_add: {
- return handleInterlockedOp(
- *this, E, CGM.getHLSLRuntime().getInterlockedAddIntrinsic(),
- "hlsl.interlocked.add");
+ // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick
+ // up the raw instruction (DXIL via DXILResourceAccess for resource
+ // pointers, SPIR-V via selectAtomicRMW). No intermediate intrinsic.
+ return handleInterlockedAdd(*this, E);
}
case Builtin::BI__builtin_hlsl_interlocked_or: {
return handleInterlockedOp(*this, E,
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index cf47b1633fd3c..5ca73dc91d75e 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -151,7 +151,6 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and)
- GENERATE_HLSL_INTRINSIC_FUNCTION(InterlockedAdd, interlocked_add)
GENERATE_HLSL_INTRINSIC_FUNCTION(InterlockedOr, interlocked_or)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
index 245487b71875f..07cbab6d85766 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
@@ -1640,24 +1640,17 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() {
assert(!Record->isCompleteDefinition() && "record is already complete");
ASTContext &AST = SemaRef.getASTContext();
- // Each entry declares two overloads (with and without an out original-value
- // parameter). Adding a new atomic here only requires a new line — the shared
- // helper takes care of the composition.
+ // This is a helper that declares two overloads with and without an out
+ // original-value parameter for each entry.
addByteAddressBufferInterlockedMethod("InterlockedAdd", AST.UnsignedIntTy,
"__builtin_hlsl_interlocked_add");
- // 64-bit typed atomics on UAVs require SM 6.6 (DXIL 1.6 introduces the
- // int64 overload of the atomicBinOp op). Skip synthesizing the *64 methods
- // on older DXIL targets so callers get "no matching member function" from
- // overload resolution — this matches DXC and mirrors how other HLSL SM-gated
- // features are handled (see hlsl_intrinsics.h `_HLSL_AVAILABILITY`
- // annotations). Non-DXIL targets (e.g., SPIR-V) always get the method: their
- // 64-bit atomic support is gated by device extensions, not shader model.
+ // Skip synthesizing the 64 bit methods on DXIL targets older than SM 6.6.
const llvm::Triple &TT = AST.getTargetInfo().getTriple();
- bool DXILNeedsSM66 =
- TT.getArch() == llvm::Triple::dxil &&
- AST.getTargetInfo().getPlatformMinVersion() < VersionTuple(6, 6);
- if (!DXILNeedsSM66) {
+ bool HasInt64AtomicSupport =
+ TT.getArch() != llvm::Triple::dxil ||
+ AST.getTargetInfo().getPlatformMinVersion() >= VersionTuple(6, 6);
+ if (HasInt64AtomicSupport) {
// HLSL's uint64_t is `unsigned long`.
addByteAddressBufferInterlockedMethod("InterlockedAdd64",
AST.UnsignedLongTy,
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 4cedfe893d91c..97913761566b2 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -4590,26 +4590,17 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
return true;
}
- // 64-bit interlocked ops require SM 6.6 on DXIL — the DXIL 1.6 int64
- // overloads of atomicBinOp/cmpXchg are what enable them. The synthesized
- // wrapper methods (e.g. RWByteAddressBuffer::InterlockedAdd64) that call
- // this builtin are themselves only declared when the target supports it
- // (see HLSLBuiltinTypeDeclBuilder), so pre-SM6.6 usage is caught by
- // overload resolution. This defensive check catches direct
- // `__builtin_hlsl_interlocked_add` calls from HLSL code with a 64-bit
- // dest on pre-SM6.6 DXIL targets. Skip synthetic invocations (invalid
- // source location) built while composing wrapper method bodies.
+ // 64-bit interlocked ops require SM 6.6 on DXIL. The synthesized wrapper
+ // methods (e.g. RWByteAddressBuffer::InterlockedAdd64) are only declared
+ // on SM 6.6+, so this defensive check only fires for direct builtin
+ // calls; skip synthetic invocations (invalid source location).
const TargetInfo &TI = SemaRef.Context.getTargetInfo();
if (TheCall->getBeginLoc().isValid() &&
TI.getTriple().getArch() == llvm::Triple::dxil &&
SemaRef.Context.getTypeSize(DestTy) == 64 &&
TI.getPlatformMinVersion() < VersionTuple(6, 6)) {
- llvm::StringRef PlatformName(
- AvailabilityAttr::getPrettyPlatformName(TI.getPlatformName()));
- SemaRef.Diag(TheCall->getBeginLoc(), diag::warn_hlsl_availability)
- << TheCall->getDirectCallee() << PlatformName
- << VersionTuple(6, 6).getAsString() << /*UseEnvironment=*/false
- << /*EnvName=*/"";
+ SemaRef.Diag(TheCall->getBeginLoc(), diag::err_hlsl_builtin_requires_sm)
+ << TheCall->getDirectCallee() << VersionTuple(6, 6).getAsString();
return true;
}
diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
index ae6179dc4bc99..a7306249e21c7 100644
--- a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
@@ -11,69 +11,69 @@
RWByteAddressBuffer BAB : register(u0);
RasterizerOrderedByteAddressBuffer ROVB : register(u1);
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int_2arg
+// CHECK-LABEL: define void @{{.*}}test_bab_int_2arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic
export void test_bab_int_2arg(uint off, int v) {
BAB.InterlockedAdd(off, v);
}
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint_3arg
+// CHECK-LABEL: define void @{{.*}}test_bab_uint_3arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic
// DXCHECK: store i32 %[[R]], ptr {{.*}}
export void test_bab_uint_3arg(uint off, uint v, out uint orig) {
BAB.InterlockedAdd(off, v, orig);
}
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int_2arg
+// CHECK-LABEL: define void @{{.*}}test_rovb_int_2arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic
export void test_rovb_int_2arg(uint off, int v) {
ROVB.InterlockedAdd(off, v);
}
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint_3arg
+// CHECK-LABEL: define void @{{.*}}test_rovb_uint_3arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic
// DXCHECK: store i32 %[[R]], ptr {{.*}}
export void test_rovb_uint_3arg(uint off, uint v, out uint orig) {
ROVB.InterlockedAdd(off, v, orig);
}
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int64_2arg
+// CHECK-LABEL: define void @{{.*}}test_bab_int64_2arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic
export void test_bab_int64_2arg(uint off, int64_t v) {
BAB.InterlockedAdd64(off, v);
}
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64_3arg
+// CHECK-LABEL: define void @{{.*}}test_bab_uint64_3arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic
// DXCHECK: store i64 %[[R]], ptr {{.*}}
export void test_bab_uint64_3arg(uint off, uint64_t v, out uint64_t orig) {
BAB.InterlockedAdd64(off, v, orig);
}
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int64_2arg
+// CHECK-LABEL: define void @{{.*}}test_rovb_int64_2arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic
export void test_rovb_int64_2arg(uint off, int64_t v) {
ROVB.InterlockedAdd64(off, v);
}
-// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint64_3arg
+// CHECK-LABEL: define void @{{.*}}test_rovb_uint64_3arg
// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
-// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic
// DXCHECK: store i64 %[[R]], ptr {{.*}}
export void test_rovb_uint64_3arg(uint off, uint64_t v, out uint64_t orig) {
ROVB.InterlockedAdd64(off, v, orig);
diff --git a/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl
index da53bba3e0d05..6bb4fd25c4a0c 100644
--- a/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl
@@ -14,45 +14,45 @@ groupshared int64_t gs_i64;
groupshared uint64_t gs_u64;
// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int_2arg
-// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}})
-// SPVCHECK: call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}})
+// DXCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic
+// SPVCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic
export void test_int_2arg(int v) {
InterlockedAdd(gs_i32, v);
}
// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint_2arg
-// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}})
-// SPVCHECK: call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}})
+// DXCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic
+// SPVCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic
export void test_uint_2arg(uint v) {
InterlockedAdd(gs_u32, v);
}
// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int_3arg
-// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}})
-// SPVCHECK: %[[R:.*]] = call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic
+// SPVCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic
// CHECK: store i32 %[[R]], ptr {{.*}}
export void test_int_3arg(int v, out int orig) {
InterlockedAdd(gs_i32, v, orig);
}
// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint_3arg
-// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}})
-// SPVCHECK: %[[R:.*]] = call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic
+// SPVCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic
// CHECK: store i32 %[[R]], ptr {{.*}}
export void test_uint_3arg(uint v, out uint orig) {
InterlockedAdd(gs_u32, v, orig);
}
// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int64_2arg
-// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}})
-// SPVCHECK: call spir_func i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}})
+// DXCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}} monotonic
+// SPVCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}} monotonic
export void test_int64_2arg(int64_t v) {
InterlockedAdd(gs_i64, v);
}
// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint64_3arg
-// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}})
-// SPVCHECK: %[[R:.*]] = call spir_func i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}} monotonic
+// SPVCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}} monotonic
// CHECK: store i64 %[[R]], ptr {{.*}}
export void test_uint64_3arg(uint64_t v, out uint64_t orig) {
InterlockedAdd(gs_u64, v, orig);
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
index 13fc70d700421..234932366e406 100644
--- a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
@@ -2,37 +2,11 @@
// RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \
// RUN: -verify-ignore-unexpected=note,warning
-// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
-// RUN: -triple dxil-pc-shadermodel6.5-compute -DTEST_SM65 %s -fsyntax-only \
-// RUN: -verify -verify-ignore-unexpected=note,warning
-
RWByteAddressBuffer BAB : register(u0);
RasterizerOrderedByteAddressBuffer ROVB : register(u1);
struct S { int x; };
-#ifdef TEST_SM65
-
-// InterlockedAdd64 is only synthesized on DXIL when the shader model is at
-// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so
-// the reference must fail with "no member named".
-void sm65_no_bab_add64(uint off, int64_t v) {
- BAB.InterlockedAdd64(off, v);
- // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}}
-}
-
-void sm65_no_rovb_add64(uint off, int64_t v) {
- ROVB.InterlockedAdd64(off, v);
- // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}}
-}
-
-// 32-bit InterlockedAdd is always available.
-void sm65_bab_add32_ok(uint off, int v) {
- BAB.InterlockedAdd(off, v);
-}
-
-#else
-
void too_few(uint off) {
BAB.InterlockedAdd(off);
// expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
@@ -59,5 +33,3 @@ void rovb_struct_value(uint off, S v) {
ROVB.InterlockedAdd(off, v);
// expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
}
-
-#endif
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl
new file mode 100644
index 0000000000000..5e51cbf36b32b
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.5-compute %s -fsyntax-only -verify \
+// RUN: -verify-ignore-unexpected=note,warning
+
+RWByteAddressBuffer BAB : register(u0);
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+
+// InterlockedAdd64 is only synthesized on DXIL when the shader model is at
+// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so
+// the reference must fail with "no member named".
+void sm65_no_bab_add64(uint off, int64_t v) {
+ BAB.InterlockedAdd64(off, v);
+ // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}}
+}
+
+void sm65_no_rovb_add64(uint off, int64_t v) {
+ ROVB.InterlockedAdd64(off, v);
+ // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}}
+}
+
+// 32-bit InterlockedAdd is always available.
+void sm65_bab_add32_ok(uint off, int v) {
+ BAB.InterlockedAdd(off, v);
+}
+
+// Direct calls to the 64-bit interlocked builtin must also be rejected with a
+// clear source-location error on pre-SM6.6 DXIL targets.
+groupshared int64_t gs_i64;
+void sm65_direct_builtin(int64_t v) {
+ __builtin_hlsl_interlocked_add(gs_i64, v);
+ // expected-error at -1 {{'__builtin_hlsl_interlocked_add' requires shader model 6.6 or newer}}
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index f1b3845ea1efa..ec0e8d156b40a 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -61,15 +61,6 @@ def int_dx_resource_store_rawbuffer
: DefaultAttrsIntrinsic<
[], [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, llvm_any_ty],
[IntrWriteMem]>;
-// Resource atomic binary op: performs an atomic read-modify-write on a UAV
-// resource element and returns the original value. The i32 operation code
-// matches DXIL's AtomicBinOpCode enum.
-def int_dx_resource_atomicbinop
- : DefaultAttrsIntrinsic<[llvm_anyint_ty],
- [llvm_any_ty, llvm_i32_ty, llvm_i32_ty,
- llvm_i32_ty, LLVMMatchType<0>],
- [IntrArgMemOnly]>;
-
// dx.resource.load.cbufferrow encodes the number of elements returned in the
// function name. The total size of the return should always be 128 bits.
def int_dx_resource_load_cbufferrow_8
@@ -265,10 +256,6 @@ def int_dx_wave_getlaneindex : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrCon
def int_dx_wave_reduce_or : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>;
def int_dx_wave_reduce_xor : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>;
def int_dx_wave_reduce_and : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>;
-def int_dx_interlocked_add :
- DefaultAttrsIntrinsic<[llvm_anyint_ty],
- [llvm_anyptr_ty, LLVMMatchType<0>],
- [IntrArgMemOnly]>;
def int_dx_interlocked_or :
DefaultAttrsIntrinsic<[llvm_anyint_ty],
[llvm_anyptr_ty, LLVMMatchType<0>],
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index d948ef78b9584..6a2a5a5cbfc42 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -148,10 +148,6 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]
def int_spv_wave_reduce_or : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_reduce_xor : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_reduce_and : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
- def int_spv_interlocked_add :
- DefaultAttrsIntrinsic<[llvm_anyint_ty],
- [llvm_anyptr_ty, LLVMMatchType<0>],
- [IntrArgMemOnly]>;
def int_spv_interlocked_or :
DefaultAttrsIntrinsic<[llvm_anyint_ty],
[llvm_anyptr_ty, LLVMMatchType<0>],
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index 1025015c09a1f..f6c316f883a54 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -228,7 +228,6 @@ static bool isIntrinsicExpansion(Function &F) {
case Intrinsic::dx_sign:
case Intrinsic::dx_step:
case Intrinsic::dx_radians:
- case Intrinsic::dx_interlocked_add:
case Intrinsic::dx_interlocked_or:
case Intrinsic::usub_sat:
case Intrinsic::vector_reduce_add:
@@ -1341,9 +1340,6 @@ static bool expandIntrinsic(Function &F, CallInst *Orig) {
case Intrinsic::dx_radians:
Result = expandRadiansIntrinsic(Orig);
break;
- case Intrinsic::dx_interlocked_add:
- Result = expandInterlockedIntrinsic(Orig, AtomicRMWInst::Add);
- break;
case Intrinsic::dx_interlocked_or:
Result = expandInterlockedIntrinsic(Orig, AtomicRMWInst::Or);
break;
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 64e0de8e55bdd..448a7d4e48736 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -187,6 +187,19 @@ class OpLowerer {
SmallVector<CallInst *> ToRemove;
SmallVector<Function *> CastFns;
+ // Also pick up any `dx.resource.casthandle` calls that were introduced
+ // outside of this pass (e.g. by DXILResourceAccess when it emits DXIL
+ // ops directly). All such casts must be resolved here.
+ for (Function &F : M) {
+ if (!F.isDeclaration() ||
+ F.getIntrinsicID() != Intrinsic::dx_resource_casthandle)
+ continue;
+ for (User *U : F.users())
+ if (auto *CI = dyn_cast<CallInst>(U))
+ if (!llvm::is_contained(CleanupCasts, CI))
+ CleanupCasts.push_back(CI);
+ }
+
for (CallInst *Cast : CleanupCasts) {
// These casts were only put in to ease the move from `target("dx")` types
// to `dx.types.Handle in a piecemeal way. At this point, all of the
@@ -849,33 +862,6 @@ class OpLowerer {
});
}
- [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) {
- IRBuilder<> &IRB = OpBuilder.getIRB();
-
- return replaceFunction(F, [&](CallInst *CI) -> Error {
- IRB.SetInsertPoint(CI);
- Value *Handle =
- createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());
- Value *Index = CI->getArgOperand(1);
- Value *Offset = CI->getArgOperand(2);
- Value *BinOp = CI->getArgOperand(3);
- Value *NewValue = CI->getArgOperand(4);
-
- std::array<Value *, 6> Args{Handle, BinOp, Index,
- Offset, IRB.getInt32(0), NewValue};
-
- Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
- OpCode::AtomicBinOp, Args, CI->getName(), CI->getType());
-
- if (Error E = OpCall.takeError())
- return E;
-
- CI->replaceAllUsesWith(*OpCall);
- CI->eraseFromParent();
- return Error::success();
- });
- }
-
[[nodiscard]] bool lowerGetDimensionsX(Function &F) {
IRBuilder<> &IRB = OpBuilder.getIRB();
Type *Int32Ty = IRB.getInt32Ty();
@@ -1237,9 +1223,6 @@ class OpLowerer {
case Intrinsic::dx_resource_updatecounter:
HasErrors |= lowerUpdateCounter(F);
break;
- case Intrinsic::dx_resource_atomicbinop:
- HasErrors |= lowerResourceAtomicBinOp(F);
- break;
case Intrinsic::dx_resource_getdimensions_x:
HasErrors |= lowerGetDimensionsX(F);
break;
diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
index 5f5c4660e744b..f9d7e8f85b198 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "DXILResourceAccess.h"
+#include "DXILOpBuilder.h"
#include "DirectX.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/DXILResource.h"
@@ -272,13 +273,21 @@ static std::optional<unsigned> getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) {
}
static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
- dxil::ResourceTypeInfo &RTI) {
+ dxil::ResourceTypeInfo &RTI,
+ std::optional<dxil::DXILOpBuilder> &OpBuilder) {
std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation());
if (!BinOpCode) {
+ // TODO(#nnn): DXIL only defines atomic ops for Add/And/Or/Xor/
+ // Min/Max/UMin/UMax/Xchg; the remaining atomicrmw ops (FSub, Nand,
+ // FAdd, FMin/Max variants, UIncWrap, etc.) have no direct DXIL
+ // equivalent and need explicit expansion (e.g. compare-and-swap loop).
reportFatalUsageError("DXIL resource atomicrmw operation not implemented");
return;
}
+ if (!OpBuilder)
+ OpBuilder.emplace(*AI->getModule());
+
const DataLayout &DL = AI->getDataLayout();
IRBuilder<> Builder(AI);
Value *Index = II->getOperand(1);
@@ -298,19 +307,37 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
}
auto *BinOp = Builder.getInt32(*BinOpCode);
- Value *V = Builder.CreateIntrinsic(
- AI->getType(), Intrinsic::dx_resource_atomicbinop,
- {II->getOperand(0), Index, Offset, BinOp, AI->getValOperand()});
- AI->replaceAllUsesWith(V);
+
+ // Cast the target-extension typed handle to `%dx.types.Handle` so we can
+ // emit the DXIL op directly. DXILOpLowering::cleanupHandleCasts will
+ // reconcile this cast once the handle-defining intrinsic has been lowered.
+ Value *Handle = Builder.CreateIntrinsic(OpBuilder->getHandleType(),
+ Intrinsic::dx_resource_casthandle,
+ {II->getOperand(0)});
+
+ std::array<Value *, 6> Args{
+ Handle, BinOp, Index, Offset, Builder.getInt32(0), AI->getValOperand()};
+
+ OpBuilder->getIRB().SetInsertPoint(AI);
+ Expected<CallInst *> OpCall = OpBuilder->tryCreateOp(
+ dxil::OpCode::AtomicBinOp, Args, AI->getName(), AI->getType());
+ if (Error E = OpCall.takeError()) {
+ AI->getContext().emitError(AI, toString(std::move(E)));
+ return;
+ }
+
+ AI->replaceAllUsesWith(*OpCall);
}
-static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
- dxil::ResourceTypeInfo &RTI) {
+static void
+createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
+ dxil::ResourceTypeInfo &RTI,
+ std::optional<dxil::DXILOpBuilder> &OpBuilder) {
switch (RTI.getResourceKind()) {
case dxil::ResourceKind::TypedBuffer:
case dxil::ResourceKind::RawBuffer:
case dxil::ResourceKind::StructuredBuffer:
- return createAtomicBinOp(II, AI, RTI);
+ return createAtomicBinOp(II, AI, RTI, OpBuilder);
case dxil::ResourceKind::Texture1D:
case dxil::ResourceKind::Texture2D:
case dxil::ResourceKind::Texture2DMS:
@@ -322,12 +349,15 @@ static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
case dxil::ResourceKind::TextureCubeArray:
case dxil::ResourceKind::FeedbackTexture2D:
case dxil::ResourceKind::FeedbackTexture2DArray:
+ // TODO(#nnn): lower atomicrmw on texture UAVs to dx.op.textureAtomic.
reportFatalUsageError(
"DXIL atomicrmw not implemented for texture resources");
return;
case dxil::ResourceKind::CBuffer:
case dxil::ResourceKind::Sampler:
case dxil::ResourceKind::TBuffer:
+ // TODO(#nnn): decide whether these resource kinds should be diagnosed
+ // in the frontend instead of reaching backend lowering.
reportFatalUsageError(
"DXIL atomicrmw not implemented for this resource type");
return;
@@ -874,7 +904,8 @@ static bool legalizeResourceHandles(Function &F, DXILResourceTypeMap &DRTM) {
return MadeChanges;
}
-static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
+static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI,
+ std::optional<dxil::DXILOpBuilder> &OpBuilder) {
SmallVector<User *> Worklist;
for (User *U : II->users())
Worklist.push_back(U);
@@ -898,35 +929,8 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
createLoadIntrinsic(II, LI, RTI);
DeadInsts.push_back(LI);
} else if (auto *AI = dyn_cast<AtomicRMWInst>(U)) {
- createAtomicBinOpIntrinsic(II, AI, RTI);
- DeadInsts.push_back(AI);
- } else if (auto *CI = dyn_cast<CallInst>(U)) {
- // `dx.interlocked.*` intrinsics wrap an atomicrmw and are expanded to
- // one by DXILIntrinsicExpansion — but that pass runs after this one, so
- // when the source of the pointer is a resource we must expand them here
- // (and immediately process the resulting atomicrmw) instead of letting
- // the pointer escape.
- auto *IntrinCall = dyn_cast<IntrinsicInst>(CI);
- std::optional<AtomicRMWInst::BinOp> Op;
- if (IntrinCall) {
- switch (IntrinCall->getIntrinsicID()) {
- case Intrinsic::dx_interlocked_add:
- Op = AtomicRMWInst::Add;
- break;
- default:
- break;
- }
- }
- if (!Op)
- llvm_unreachable("Unhandled instruction - pointer escaped?");
- IRBuilder<> Builder(IntrinCall);
- auto *AI = Builder.CreateAtomicRMW(
- *Op, IntrinCall->getArgOperand(0), IntrinCall->getArgOperand(1),
- MaybeAlign(), AtomicOrdering::Monotonic);
- IntrinCall->replaceAllUsesWith(AI);
- createAtomicBinOpIntrinsic(II, AI, RTI);
+ createAtomicBinOpIntrinsic(II, AI, RTI, OpBuilder);
DeadInsts.push_back(AI);
- DeadInsts.push_back(IntrinCall);
} else
llvm_unreachable("Unhandled instruction - pointer escaped?");
}
@@ -938,6 +942,9 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
}
static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {
+ // Constructed lazily on the first atomicrmw so that non-atomic resource
+ // access still works on triples without a DXIL version.
+ std::optional<dxil::DXILOpBuilder> OpBuilder;
SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources;
for (BasicBlock &BB : make_early_inc_range(F))
for (Instruction &I : BB)
@@ -953,7 +960,7 @@ static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {
}
for (auto &[II, RI] : Resources)
- replaceAccess(II, RI);
+ replaceAccess(II, RI, OpBuilder);
return !Resources.empty();
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 11128eadef95f..bd50f03489255 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -5440,8 +5440,6 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
case Intrinsic::spv_wave_reduce_and:
return selectWaveReduceOp(ResVReg, ResType, I,
SPIRV::OpGroupNonUniformBitwiseAnd);
- case Intrinsic::spv_interlocked_add:
- return selectInterlockedOp(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
case Intrinsic::spv_interlocked_or:
return selectInterlockedOp(ResVReg, ResType, I, SPIRV::OpAtomicOr);
case Intrinsic::spv_wave_reduce_umax:
diff --git a/llvm/test/CodeGen/DirectX/InterlockedAdd.ll b/llvm/test/CodeGen/DirectX/InterlockedAdd.ll
deleted file mode 100644
index b12490b59cb79..0000000000000
--- a/llvm/test/CodeGen/DirectX/InterlockedAdd.ll
+++ /dev/null
@@ -1,52 +0,0 @@
-; RUN: opt -S -dxil-intrinsic-expansion -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s
-
-; Verify llvm.dx.interlocked.add expands to atomicrmw add monotonic.
-
-; Groupshared (addrspace 3) memory tests.
- at gs_i32 = internal addrspace(3) global i32 zeroinitializer
- at gs_i64 = internal addrspace(3) global i64 zeroinitializer
-
-define i32 @test_i32(i32 %v) {
-entry:
-; CHECK-LABEL: @test_i32
-; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) @gs_i32, i32 %v monotonic
-; CHECK: ret i32 %[[R]]
- %r = call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) @gs_i32, i32 %v)
- ret i32 %r
-}
-
-define i64 @test_i64(i64 %v) {
-entry:
-; CHECK-LABEL: @test_i64
-; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) @gs_i64, i64 %v monotonic
-; CHECK: ret i64 %[[R]]
- %r = call i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3) @gs_i64, i64 %v)
- ret i64 %r
-}
-
-; Device (addrspace 1) memory tests.
- at dev_i32 = external addrspace(1) global i32
- at dev_i64 = external addrspace(1) global i64
-
-define i32 @test_device_i32(i32 %v) {
-entry:
-; CHECK-LABEL: @test_device_i32
-; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(1) @dev_i32, i32 %v monotonic
-; CHECK: ret i32 %[[R]]
- %r = call i32 @llvm.dx.interlocked.add.i32.p1(ptr addrspace(1) @dev_i32, i32 %v)
- ret i32 %r
-}
-
-define i64 @test_device_i64(i64 %v) {
-entry:
-; CHECK-LABEL: @test_device_i64
-; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(1) @dev_i64, i64 %v monotonic
-; CHECK: ret i64 %[[R]]
- %r = call i64 @llvm.dx.interlocked.add.i64.p1(ptr addrspace(1) @dev_i64, i64 %v)
- ret i64 %r
-}
-
-declare i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3), i32)
-declare i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3), i64)
-declare i32 @llvm.dx.interlocked.add.i32.p1(ptr addrspace(1), i32)
-declare i64 @llvm.dx.interlocked.add.i64.p1(ptr addrspace(1), i64)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll
deleted file mode 100644
index 44ff6cff6ee5f..0000000000000
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll
+++ /dev/null
@@ -1,36 +0,0 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
-
-; Test lowering of llvm.spv.interlocked.add to OpAtomicIAdd.
-
-; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#scope_wg:]] = OpConstant %[[#uint]] 2
-; CHECK-DAG: %[[#scope_dev:]] = OpConstant %[[#uint]] 1
-; CHECK-DAG: %[[#mem_wg:]] = OpConstant %[[#uint]] 256
-; CHECK-DAG: %[[#mem_uniform:]] = OpConstant %[[#uint]] 64
-
- at gs_i32 = internal addrspace(3) global i32 zeroinitializer
- at dev_i32 = external addrspace(11) global i32
-
-; Workgroup (addrspace 3) memory tests.
-
-; CHECK-LABEL: Begin function test_i32
-define i32 @test_i32(i32 %v) {
-entry:
-; CHECK: %[[#R:]] = OpAtomicIAdd %[[#uint]] %[[#]] %[[#scope_wg]] %[[#mem_wg]] %[[#]]
- %r = call i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) @gs_i32, i32 %v)
- ret i32 %r
-}
-
-; Device / StorageBuffer (addrspace 11) memory tests.
-
-; CHECK-LABEL: Begin function test_device_i32
-define i32 @test_device_i32(i32 %v) {
-entry:
-; CHECK: %[[#R:]] = OpAtomicIAdd %[[#uint]] %[[#]] %[[#scope_dev]] %[[#mem_uniform]] %[[#]]
- %r = call i32 @llvm.spv.interlocked.add.i32.p11(ptr addrspace(11) @dev_i32, i32 %v)
- ret i32 %r
-}
-
-declare i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3), i32)
-declare i32 @llvm.spv.interlocked.add.i32.p11(ptr addrspace(11), i32)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll
deleted file mode 100644
index 794c0a925862a..0000000000000
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
-
-; Test lowering of llvm.spv.interlocked.add with i64 to OpAtomicIAdd.
-
-; CHECK-DAG: %[[#ulong:]] = OpTypeInt 64 0
-; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#scope_wg:]] = OpConstant %[[#uint]] 2
-; CHECK-DAG: %[[#scope_dev:]] = OpConstant %[[#uint]] 1
-; CHECK-DAG: %[[#mem_wg:]] = OpConstant %[[#uint]] 256
-; CHECK-DAG: %[[#mem_uniform:]] = OpConstant %[[#uint]] 64
-
- at gs_i64 = internal addrspace(3) global i64 zeroinitializer
- at dev_i64 = external addrspace(11) global i64
-
-; Workgroup (addrspace 3) memory test.
-
-; CHECK-LABEL: Begin function test_i64
-define i64 @test_i64(i64 %v) {
-entry:
-; CHECK: %[[#R:]] = OpAtomicIAdd %[[#ulong]] %[[#]] %[[#scope_wg]] %[[#mem_wg]] %[[#]]
- %r = call i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3) @gs_i64, i64 %v)
- ret i64 %r
-}
-
-; Device / StorageBuffer (addrspace 11) memory test.
-
-; CHECK-LABEL: Begin function test_device_i64
-define i64 @test_device_i64(i64 %v) {
-entry:
-; CHECK: %[[#R:]] = OpAtomicIAdd %[[#ulong]] %[[#]] %[[#scope_dev]] %[[#mem_uniform]] %[[#]]
- %r = call i64 @llvm.spv.interlocked.add.i64.p11(ptr addrspace(11) @dev_i64, i64 %v)
- ret i64 %r
-}
-
-declare i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3), i64)
-declare i64 @llvm.spv.interlocked.add.i64.p11(ptr addrspace(11), i64)
>From 4e2b382d310f132bb512aa1b976f6ad7df91554c Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Thu, 16 Jul 2026 11:34:00 -0700
Subject: [PATCH 4/7] fix failing tests
---
.../ByteAddressBuffer-InterlockedAdd-errors.hlsl | 9 ++++++++-
.../ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl | 4 ++--
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
index 234932366e406..f63e64b9f5837 100644
--- a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
@@ -1,7 +1,14 @@
// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
-// RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \
+// RUN: -triple dxil-pc-shadermodel6.6-library %s -fsyntax-only -verify \
// RUN: -verify-ignore-unexpected=note,warning
+// notes that are ignored are strictly ones of the form:
+// (frontend): candidate function not viable: requires X arguments, but Y was provided
+// or
+// (frontend): candidate function not viable: no known conversion from X to Y for Nth argument
+// which is in line with expectations, but is difficult to exactly match since
+// the notes are not tied to explicit source lines, but just (frontend).
+
RWByteAddressBuffer BAB : register(u0);
RasterizerOrderedByteAddressBuffer ROVB : register(u1);
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl
index 5e51cbf36b32b..e6210ba36e357 100644
--- a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl
@@ -1,6 +1,6 @@
// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
-// RUN: -triple dxil-pc-shadermodel6.5-compute %s -fsyntax-only -verify \
-// RUN: -verify-ignore-unexpected=note,warning
+// RUN: -triple dxil-pc-shadermodel6.5-library %s -fsyntax-only -verify \
+// RUN: -verify-ignore-unexpected=warning
RWByteAddressBuffer BAB : register(u0);
RasterizerOrderedByteAddressBuffer ROVB : register(u1);
>From 02619f419b9d7d95cd16901f4d76a15696b9f69e Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Thu, 16 Jul 2026 12:50:48 -0700
Subject: [PATCH 5/7] update target profile
---
.../CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
index a7306249e21c7..cf939a7507dbf 100644
--- a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
@@ -1,5 +1,5 @@
// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \
-// RUN: dxil-pc-shadermodel6.6-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \
// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK
// Test that the RWByteAddressBuffer::InterlockedAdd and
>From 1e13e596e976c4fe1cafb5979e09aeba6934c71a Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Thu, 16 Jul 2026 14:31:58 -0700
Subject: [PATCH 6/7] try different error delivery to fix not opt
---
llvm/lib/Target/DirectX/DXILResourceAccess.cpp | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
index f9d7e8f85b198..056c0f8cd5a5a 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
@@ -322,7 +322,12 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
Expected<CallInst *> OpCall = OpBuilder->tryCreateOp(
dxil::OpCode::AtomicBinOp, Args, AI->getName(), AI->getType());
if (Error E = OpCall.takeError()) {
- AI->getContext().emitError(AI, toString(std::move(E)));
+ std::string Message(toString(std::move(E)));
+ AI->getContext().diagnose(DiagnosticInfoUnsupported(
+ *AI->getFunction(), Message, AI->getDebugLoc()));
+ // RAUW with poison so the caller's subsequent eraseFromParent() doesn't
+ // leave dangling uses of the AtomicRMWInst.
+ AI->replaceAllUsesWith(PoisonValue::get(AI->getType()));
return;
}
>From c4f6704f3c7c2841fd670537a2d04a8925013518 Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Fri, 17 Jul 2026 14:51:53 -0700
Subject: [PATCH 7/7] self review
---
clang/lib/CodeGen/CGHLSLBuiltins.cpp | 13 +++++++------
llvm/lib/Target/DirectX/DXILResourceAccess.cpp | 7 -------
2 files changed, 7 insertions(+), 13 deletions(-)
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index ef8be15fdfb40..12a3f61c734d0 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -310,17 +310,18 @@ static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF,
llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture");
}
-static Value *handleInterlockedAdd(CodeGenFunction &CGF, const CallExpr *E) {
- // Emit `atomicrmw add` directly — no intermediate `*.interlocked.add`
- // intrinsic needed on either DXIL or SPIR-V.
+static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
+ llvm::AtomicRMWInst::BinOp Op) {
+ // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on
+ // either DXIL or SPIR-V.
LValue DestLV = CGF.EmitLValue(E->getArg(0));
Address DestAddr = DestLV.getAddress();
Value *Val = CGF.EmitScalarExpr(E->getArg(1));
assert(E->getArg(1)->getType()->isIntegerType() &&
- "Intrinsic InterlockedAdd value operand must be an integer");
+ "Intrinsic InterlockedOp value operand must be an integer");
llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW(
- llvm::AtomicRMWInst::Add, DestAddr, Val, llvm::AtomicOrdering::Monotonic);
+ Op, DestAddr, Val, llvm::AtomicOrdering::Monotonic);
// The 3-arg overload writes the old value (the RMW's return value) into
// the `original_value` reference parameter.
@@ -1481,7 +1482,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
// Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick
// up the raw instruction (DXIL via DXILResourceAccess for resource
// pointers, SPIR-V via selectAtomicRMW). No intermediate intrinsic.
- return handleInterlockedAdd(*this, E);
+ return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::Add);
}
case Builtin::BI__builtin_hlsl_interlocked_or: {
return handleInterlockedOp(*this, E,
diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
index 056c0f8cd5a5a..ae3e0b5f6c503 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
@@ -277,10 +277,6 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
std::optional<dxil::DXILOpBuilder> &OpBuilder) {
std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation());
if (!BinOpCode) {
- // TODO(#nnn): DXIL only defines atomic ops for Add/And/Or/Xor/
- // Min/Max/UMin/UMax/Xchg; the remaining atomicrmw ops (FSub, Nand,
- // FAdd, FMin/Max variants, UIncWrap, etc.) have no direct DXIL
- // equivalent and need explicit expansion (e.g. compare-and-swap loop).
reportFatalUsageError("DXIL resource atomicrmw operation not implemented");
return;
}
@@ -354,15 +350,12 @@ createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
case dxil::ResourceKind::TextureCubeArray:
case dxil::ResourceKind::FeedbackTexture2D:
case dxil::ResourceKind::FeedbackTexture2DArray:
- // TODO(#nnn): lower atomicrmw on texture UAVs to dx.op.textureAtomic.
reportFatalUsageError(
"DXIL atomicrmw not implemented for texture resources");
return;
case dxil::ResourceKind::CBuffer:
case dxil::ResourceKind::Sampler:
case dxil::ResourceKind::TBuffer:
- // TODO(#nnn): decide whether these resource kinds should be diagnosed
- // in the frontend instead of reaching backend lowering.
reportFatalUsageError(
"DXIL atomicrmw not implemented for this resource type");
return;
More information about the cfe-commits
mailing list