[clang] [llvm] Add InterlockedAdd resource methods (PR #208128)
Joshua Batista via cfe-commits
cfe-commits at lists.llvm.org
Tue Jul 7 18:02:54 PDT 2026
https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/208128
This PR completes the implementation of the InterlockedAdd functions in HLSL, by adding these methods to all existing resource classes.
Fixes: https://github.com/llvm/llvm-project/issues/99122
Assisted by: Github Copilot
>From c2e34e072d1e76d164397fae6b45de750d779913 Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Tue, 7 Jul 2026 18:00:56 -0700
Subject: [PATCH] first attempt, add interlockedadd resource methods
---
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 68 +++++++++
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h | 4 +
clang/lib/Sema/HLSLExternalSemaSource.cpp | 2 +
clang/lib/Sema/SemaHLSL.cpp | 23 +++
.../ByteAddressBuffer-InterlockedAdd.hlsl | 80 ++++++++++
...teAddressBuffer-InterlockedAdd-errors.hlsl | 63 ++++++++
llvm/include/llvm/IR/IntrinsicsDirectX.td | 8 +
llvm/lib/Target/DirectX/DXIL.td | 11 ++
llvm/lib/Target/DirectX/DXILOpLowering.cpp | 30 ++++
.../lib/Target/DirectX/DXILResourceAccess.cpp | 141 ++++++++++++++++++
.../DirectX/ResourceAtomicBinOp-i64-sm65.ll | 16 ++
.../CodeGen/DirectX/ResourceAtomicBinOp.ll | 59 ++++++++
12 files changed, 505 insertions(+)
create mode 100644 clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll
create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
index f8018729b4644..245487b71875f 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
@@ -1635,6 +1635,38 @@ BuiltinTypeDeclBuilder::addByteAddressBufferStoreMethods() {
return *this;
}
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() {
+ assert(!Record->isCompleteDefinition() && "record is already complete");
+ ASTContext &AST = SemaRef.getASTContext();
+
+ // Each entry declares two overloads (with and without an out original-value
+ // parameter). Adding a new atomic here only requires a new line — the shared
+ // helper takes care of the composition.
+ addByteAddressBufferInterlockedMethod("InterlockedAdd", AST.UnsignedIntTy,
+ "__builtin_hlsl_interlocked_add");
+
+ // 64-bit typed atomics on UAVs require SM 6.6 (DXIL 1.6 introduces the
+ // int64 overload of the atomicBinOp op). Skip synthesizing the *64 methods
+ // on older DXIL targets so callers get "no matching member function" from
+ // overload resolution — this matches DXC and mirrors how other HLSL SM-gated
+ // features are handled (see hlsl_intrinsics.h `_HLSL_AVAILABILITY`
+ // annotations). Non-DXIL targets (e.g., SPIR-V) always get the method: their
+ // 64-bit atomic support is gated by device extensions, not shader model.
+ const llvm::Triple &TT = AST.getTargetInfo().getTriple();
+ bool DXILNeedsSM66 =
+ TT.getArch() == llvm::Triple::dxil &&
+ AST.getTargetInfo().getPlatformMinVersion() < VersionTuple(6, 6);
+ if (!DXILNeedsSM66) {
+ // HLSL's uint64_t is `unsigned long`.
+ addByteAddressBufferInterlockedMethod("InterlockedAdd64",
+ AST.UnsignedLongTy,
+ "__builtin_hlsl_interlocked_add");
+ }
+
+ return *this;
+}
+
BuiltinTypeDeclBuilder &
BuiltinTypeDeclBuilder::addSampleMethods(ResourceDimension Dim, bool IsArray) {
assert(!Record->isCompleteDefinition() && "record is already complete");
@@ -2356,6 +2388,42 @@ BuiltinTypeDeclBuilder::addStoreFunction(DeclarationName &Name, bool IsConst,
.finalize();
}
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethod(
+ StringRef MethodName, QualType ValueTy, StringRef BuiltinName) {
+ assert(!Record->isCompleteDefinition() && "record is already complete");
+ ASTContext &AST = SemaRef.getASTContext();
+ using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+
+ // Interlocked atomics operate on a typed slot in the buffer. Compose
+ // `resource_getpointer_typed` with the scalar `__builtin_hlsl_interlocked_*`
+ // builtin so backend lowering (DXIL and SPIR-V) can pattern-match a
+ // resource-pointer atomicrmw.
+ QualType AddrSpaceElemTy =
+ AST.getAddrSpaceQualType(ValueTy, LangAS::hlsl_device);
+ QualType ElemPtrTy = AST.getPointerType(AddrSpaceElemTy);
+
+ auto BuildOverload = [&](bool WithOriginalValue) {
+ BuiltinTypeMethodBuilder MMB(*this, MethodName, AST.VoidTy);
+ MMB.addParam("Offset", AST.UnsignedIntTy).addParam("Value", ValueTy);
+ if (WithOriginalValue)
+ MMB.addParam("OriginalValue", ValueTy,
+ HLSLParamModifierAttr::Keyword_out);
+ MMB.callBuiltin("__builtin_hlsl_resource_getpointer_typed", ElemPtrTy,
+ PH::Handle, PH::_0, ValueTy)
+ .dereference(PH::LastStmt);
+ if (WithOriginalValue)
+ MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2);
+ else
+ MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1);
+ MMB.finalize();
+ };
+
+ BuildOverload(/*WithOriginalValue=*/false);
+ BuildOverload(/*WithOriginalValue=*/true);
+ return *this;
+}
+
BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() {
using PH = BuiltinTypeMethodBuilder::PlaceHolder;
ASTContext &AST = SemaRef.getASTContext();
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
index 09cf1fceca116..e809ef264198c 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
@@ -106,6 +106,7 @@ class BuiltinTypeDeclBuilder {
bool IsArray = false);
BuiltinTypeDeclBuilder &addByteAddressBufferLoadMethods();
BuiltinTypeDeclBuilder &addByteAddressBufferStoreMethods();
+ BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedMethods();
BuiltinTypeDeclBuilder &addSampleMethods(ResourceDimension Dim,
bool IsArray = false);
BuiltinTypeDeclBuilder &addSampleBiasMethods(ResourceDimension Dim,
@@ -134,6 +135,9 @@ class BuiltinTypeDeclBuilder {
QualType ReturnTy = QualType());
BuiltinTypeDeclBuilder &addStoreFunction(DeclarationName &Name, bool IsConst,
QualType ValueType);
+ BuiltinTypeDeclBuilder &
+ addByteAddressBufferInterlockedMethod(StringRef MethodName, QualType ValueTy,
+ StringRef BuiltinName);
BuiltinTypeDeclBuilder &addAppendMethod();
BuiltinTypeDeclBuilder &addConsumeMethod();
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index 1d2a3983a639d..4c1894d6761d2 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -645,6 +645,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
/*RawBuffer=*/true, /*HasCounter=*/false)
.addByteAddressBufferLoadMethods()
.addByteAddressBufferStoreMethods()
+ .addByteAddressBufferInterlockedMethods()
.addGetDimensionsMethodForBuffer()
.completeDefinition();
});
@@ -654,6 +655,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
onCompletion(Decl, [this](CXXRecordDecl *Decl) {
setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, /*IsROV=*/true,
/*RawBuffer=*/true, /*HasCounter=*/false)
+ .addByteAddressBufferInterlockedMethods()
.addGetDimensionsMethodForBuffer()
.completeDefinition();
});
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index ae6cbce2f890c..1ffddeac080fa 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -4568,6 +4568,29 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
return true;
}
+ // 64-bit interlocked ops require SM 6.6 on DXIL — the DXIL 1.6 int64
+ // overloads of atomicBinOp/cmpXchg are what enable them. The synthesized
+ // wrapper methods (e.g. RWByteAddressBuffer::InterlockedAdd64) that call
+ // this builtin are themselves only declared when the target supports it
+ // (see HLSLBuiltinTypeDeclBuilder), so pre-SM6.6 usage is caught by
+ // overload resolution. This defensive check catches direct
+ // `__builtin_hlsl_interlocked_add` calls from HLSL code with a 64-bit
+ // dest on pre-SM6.6 DXIL targets. Skip synthetic invocations (invalid
+ // source location) built while composing wrapper method bodies.
+ const TargetInfo &TI = SemaRef.Context.getTargetInfo();
+ if (TheCall->getBeginLoc().isValid() &&
+ TI.getTriple().getArch() == llvm::Triple::dxil &&
+ SemaRef.Context.getTypeSize(DestTy) == 64 &&
+ TI.getPlatformMinVersion() < VersionTuple(6, 6)) {
+ llvm::StringRef PlatformName(
+ AvailabilityAttr::getPrettyPlatformName(TI.getPlatformName()));
+ SemaRef.Diag(TheCall->getBeginLoc(), diag::warn_hlsl_availability)
+ << TheCall->getDirectCallee() << PlatformName
+ << VersionTuple(6, 6).getAsString() << /*UseEnvironment=*/false
+ << /*EnvName=*/"";
+ return true;
+ }
+
if (CheckModifiableLValue(&SemaRef, TheCall, 0))
return true;
diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
new file mode 100644
index 0000000000000..ae6179dc4bc99
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl
@@ -0,0 +1,80 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \
+// RUN: dxil-pc-shadermodel6.6-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK
+
+// Test that the RWByteAddressBuffer::InterlockedAdd and
+// RasterizerOrderedByteAddressBuffer::InterlockedAdd member methods lower to
+// `dx.resource.getpointer.typed -> dx.interlocked.add`, and that the
+// 3-argument overload stores the returned original value through the out
+// parameter.
+
+RWByteAddressBuffer BAB : register(u0);
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+export void test_bab_int_2arg(uint off, int v) {
+ BAB.InterlockedAdd(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: store i32 %[[R]], ptr {{.*}}
+export void test_bab_uint_3arg(uint off, uint v, out uint orig) {
+ BAB.InterlockedAdd(off, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+export void test_rovb_int_2arg(uint off, int v) {
+ ROVB.InterlockedAdd(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}})
+// DXCHECK: store i32 %[[R]], ptr {{.*}}
+export void test_rovb_uint_3arg(uint off, uint v, out uint orig) {
+ ROVB.InterlockedAdd(off, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int64_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+export void test_bab_int64_2arg(uint off, int64_t v) {
+ BAB.InterlockedAdd64(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: store i64 %[[R]], ptr {{.*}}
+export void test_bab_uint64_3arg(uint off, uint64_t v, out uint64_t orig) {
+ BAB.InterlockedAdd64(off, v, orig);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int64_2arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+export void test_rovb_int64_2arg(uint off, int64_t v) {
+ ROVB.InterlockedAdd64(off, v);
+}
+
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint64_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}})
+// DXCHECK: store i64 %[[R]], ptr {{.*}}
+export void test_rovb_uint64_3arg(uint off, uint64_t v, out uint64_t orig) {
+ ROVB.InterlockedAdd64(off, v, orig);
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
new file mode 100644
index 0000000000000..13fc70d700421
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \
+// RUN: -verify-ignore-unexpected=note,warning
+
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.5-compute -DTEST_SM65 %s -fsyntax-only \
+// RUN: -verify -verify-ignore-unexpected=note,warning
+
+RWByteAddressBuffer BAB : register(u0);
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+
+struct S { int x; };
+
+#ifdef TEST_SM65
+
+// InterlockedAdd64 is only synthesized on DXIL when the shader model is at
+// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so
+// the reference must fail with "no member named".
+void sm65_no_bab_add64(uint off, int64_t v) {
+ BAB.InterlockedAdd64(off, v);
+ // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}}
+}
+
+void sm65_no_rovb_add64(uint off, int64_t v) {
+ ROVB.InterlockedAdd64(off, v);
+ // expected-error at -1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}}
+}
+
+// 32-bit InterlockedAdd is always available.
+void sm65_bab_add32_ok(uint off, int v) {
+ BAB.InterlockedAdd(off, v);
+}
+
+#else
+
+void too_few(uint off) {
+ BAB.InterlockedAdd(off);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+void too_many(uint off, int v, int extra) {
+ int orig;
+ BAB.InterlockedAdd(off, v, orig, extra);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+void struct_value(uint off, S v) {
+ BAB.InterlockedAdd(off, v);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+// Same shape of errors on RasterizerOrderedByteAddressBuffer.
+void rovb_too_few(uint off) {
+ ROVB.InterlockedAdd(off);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+void rovb_struct_value(uint off, S v) {
+ ROVB.InterlockedAdd(off, v);
+ // expected-error at -1 {{no matching member function for call to 'InterlockedAdd'}}
+}
+
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 63186ab4d7191..6b88d5c633621 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -61,6 +61,14 @@ def int_dx_resource_store_rawbuffer
: DefaultAttrsIntrinsic<
[], [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, llvm_any_ty],
[IntrWriteMem]>;
+// Resource atomic binary op: performs an atomic read-modify-write on a UAV
+// resource element and returns the original value. The i32 operation code
+// matches DXIL's AtomicBinOpCode enum.
+def int_dx_resource_atomicbinop
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [llvm_any_ty, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, LLVMMatchType<0>],
+ [IntrArgMemOnly]>;
// dx.resource.load.cbufferrow encodes the number of elements returned in the
// function name. The total size of the return should always be 128 bits.
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 299d2d113b6bf..637fc05b3b2da 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -983,6 +983,17 @@ def BufferStore : DXILOp<69, bufferStore> {
let stages = [Stages<DXIL1_0, [all_stages]>];
}
+def AtomicBinOp : DXILOp<78, atomicBinOp> {
+ let Doc = "performs an atomic read-modify-write on a UAV resource "
+ "element, returning the original value";
+ // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue
+ let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy];
+ let result = OverloadTy;
+ let overloads = [Overloads<DXIL1_0, [Int32Ty]>,
+ Overloads<DXIL1_6, [Int32Ty, Int64Ty]>];
+ let stages = [Stages<DXIL1_0, [all_stages]>];
+}
+
def UpdateCounter : DXILOp<70, bufferUpdateCounter> {
let Doc = "increments/decrements a buffer counter";
let arguments = [HandleTy, Int8Ty];
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 93d5a08a6e0a2..64e0de8e55bdd 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -849,6 +849,33 @@ class OpLowerer {
});
}
+ [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) {
+ IRBuilder<> &IRB = OpBuilder.getIRB();
+
+ return replaceFunction(F, [&](CallInst *CI) -> Error {
+ IRB.SetInsertPoint(CI);
+ Value *Handle =
+ createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());
+ Value *Index = CI->getArgOperand(1);
+ Value *Offset = CI->getArgOperand(2);
+ Value *BinOp = CI->getArgOperand(3);
+ Value *NewValue = CI->getArgOperand(4);
+
+ std::array<Value *, 6> Args{Handle, BinOp, Index,
+ Offset, IRB.getInt32(0), NewValue};
+
+ Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
+ OpCode::AtomicBinOp, Args, CI->getName(), CI->getType());
+
+ if (Error E = OpCall.takeError())
+ return E;
+
+ CI->replaceAllUsesWith(*OpCall);
+ CI->eraseFromParent();
+ return Error::success();
+ });
+ }
+
[[nodiscard]] bool lowerGetDimensionsX(Function &F) {
IRBuilder<> &IRB = OpBuilder.getIRB();
Type *Int32Ty = IRB.getInt32Ty();
@@ -1210,6 +1237,9 @@ class OpLowerer {
case Intrinsic::dx_resource_updatecounter:
HasErrors |= lowerUpdateCounter(F);
break;
+ case Intrinsic::dx_resource_atomicbinop:
+ HasErrors |= lowerResourceAtomicBinOp(F);
+ break;
case Intrinsic::dx_resource_getdimensions_x:
HasErrors |= lowerGetDimensionsX(F);
break;
diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
index 25d860e615c17..5f5c4660e744b 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
@@ -26,6 +26,7 @@
#include "llvm/InitializePasses.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
+#include <optional>
#define DEBUG_TYPE "dxil-resource-access"
@@ -230,6 +231,114 @@ static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,
llvm_unreachable("Unhandled case in switch");
}
+static std::optional<unsigned> getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) {
+ switch (BinOp) {
+ case AtomicRMWInst::Add:
+ return 0;
+ case AtomicRMWInst::And:
+ return 1;
+ case AtomicRMWInst::Or:
+ return 2;
+ case AtomicRMWInst::Xor:
+ return 3;
+ case AtomicRMWInst::Min:
+ return 4;
+ case AtomicRMWInst::Max:
+ return 5;
+ case AtomicRMWInst::UMin:
+ return 6;
+ case AtomicRMWInst::UMax:
+ return 7;
+ case AtomicRMWInst::Xchg:
+ return 8;
+ case AtomicRMWInst::Sub:
+ case AtomicRMWInst::Nand:
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ case AtomicRMWInst::FMax:
+ case AtomicRMWInst::FMin:
+ case AtomicRMWInst::FMaximum:
+ case AtomicRMWInst::FMinimum:
+ case AtomicRMWInst::FMaximumNum:
+ case AtomicRMWInst::FMinimumNum:
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
+ case AtomicRMWInst::BAD_BINOP:
+ return std::nullopt;
+ }
+ llvm_unreachable("Unhandled atomicrmw operation");
+}
+
+static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
+ dxil::ResourceTypeInfo &RTI) {
+ std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation());
+ if (!BinOpCode) {
+ reportFatalUsageError("DXIL resource atomicrmw operation not implemented");
+ return;
+ }
+
+ const DataLayout &DL = AI->getDataLayout();
+ IRBuilder<> Builder(AI);
+ Value *Index = II->getOperand(1);
+
+ // The offset for the rawbuffer load/store/atomic ops is always in bytes.
+ uint64_t AccessSize = 1;
+ Value *Offset =
+ traverseGEPOffsets(DL, Builder, AI->getPointerOperand(), AccessSize);
+
+ // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
+ // entirely into the index.
+ if (!RTI.isStruct()) {
+ auto *ConstantOffset = dyn_cast<ConstantInt>(Offset);
+ if (!ConstantOffset || !ConstantOffset->isZero())
+ Index = Builder.CreateAdd(Index, Offset);
+ Offset = llvm::PoisonValue::get(Builder.getInt32Ty());
+ }
+
+ auto *BinOp = Builder.getInt32(*BinOpCode);
+ Value *V = Builder.CreateIntrinsic(
+ AI->getType(), Intrinsic::dx_resource_atomicbinop,
+ {II->getOperand(0), Index, Offset, BinOp, AI->getValOperand()});
+ AI->replaceAllUsesWith(V);
+}
+
+static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
+ dxil::ResourceTypeInfo &RTI) {
+ switch (RTI.getResourceKind()) {
+ case dxil::ResourceKind::TypedBuffer:
+ case dxil::ResourceKind::RawBuffer:
+ case dxil::ResourceKind::StructuredBuffer:
+ return createAtomicBinOp(II, AI, RTI);
+ case dxil::ResourceKind::Texture1D:
+ case dxil::ResourceKind::Texture2D:
+ case dxil::ResourceKind::Texture2DMS:
+ case dxil::ResourceKind::Texture3D:
+ case dxil::ResourceKind::TextureCube:
+ case dxil::ResourceKind::Texture1DArray:
+ case dxil::ResourceKind::Texture2DArray:
+ case dxil::ResourceKind::Texture2DMSArray:
+ case dxil::ResourceKind::TextureCubeArray:
+ case dxil::ResourceKind::FeedbackTexture2D:
+ case dxil::ResourceKind::FeedbackTexture2DArray:
+ reportFatalUsageError(
+ "DXIL atomicrmw not implemented for texture resources");
+ return;
+ case dxil::ResourceKind::CBuffer:
+ case dxil::ResourceKind::Sampler:
+ case dxil::ResourceKind::TBuffer:
+ reportFatalUsageError(
+ "DXIL atomicrmw not implemented for this resource type");
+ return;
+ case dxil::ResourceKind::RTAccelerationStructure:
+ case dxil::ResourceKind::Invalid:
+ case dxil::ResourceKind::NumEntries:
+ llvm_unreachable("Invalid resource kind for atomicrmw");
+ }
+ llvm_unreachable("Unhandled case in switch");
+}
+
static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,
dxil::ResourceTypeInfo &RTI) {
const DataLayout &DL = LI->getDataLayout();
@@ -550,6 +659,8 @@ static Instruction *getStoreLoadPointerOperand(Instruction *AI) {
return dyn_cast<Instruction>(LI->getPointerOperand());
if (auto *SI = dyn_cast<StoreInst>(AI))
return dyn_cast<Instruction>(SI->getPointerOperand());
+ if (auto *RMWI = dyn_cast<AtomicRMWInst>(AI))
+ return dyn_cast<Instruction>(RMWI->getPointerOperand());
return nullptr;
}
@@ -786,6 +897,36 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
} else if (auto *LI = dyn_cast<LoadInst>(U)) {
createLoadIntrinsic(II, LI, RTI);
DeadInsts.push_back(LI);
+ } else if (auto *AI = dyn_cast<AtomicRMWInst>(U)) {
+ createAtomicBinOpIntrinsic(II, AI, RTI);
+ DeadInsts.push_back(AI);
+ } else if (auto *CI = dyn_cast<CallInst>(U)) {
+ // `dx.interlocked.*` intrinsics wrap an atomicrmw and are expanded to
+ // one by DXILIntrinsicExpansion — but that pass runs after this one, so
+ // when the source of the pointer is a resource we must expand them here
+ // (and immediately process the resulting atomicrmw) instead of letting
+ // the pointer escape.
+ auto *IntrinCall = dyn_cast<IntrinsicInst>(CI);
+ std::optional<AtomicRMWInst::BinOp> Op;
+ if (IntrinCall) {
+ switch (IntrinCall->getIntrinsicID()) {
+ case Intrinsic::dx_interlocked_add:
+ Op = AtomicRMWInst::Add;
+ break;
+ default:
+ break;
+ }
+ }
+ if (!Op)
+ llvm_unreachable("Unhandled instruction - pointer escaped?");
+ IRBuilder<> Builder(IntrinCall);
+ auto *AI = Builder.CreateAtomicRMW(
+ *Op, IntrinCall->getArgOperand(0), IntrinCall->getArgOperand(1),
+ MaybeAlign(), AtomicOrdering::Monotonic);
+ IntrinCall->replaceAllUsesWith(AI);
+ createAtomicBinOpIntrinsic(II, AI, RTI);
+ DeadInsts.push_back(AI);
+ DeadInsts.push_back(IntrinCall);
} else
llvm_unreachable("Unhandled instruction - pointer escaped?");
}
diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll
new file mode 100644
index 0000000000000..6f5a40e0b6c2a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll
@@ -0,0 +1,16 @@
+; RUN: not opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.5-compute %s 2>&1 | FileCheck %s
+
+; Verify resource i64 atomicrmw rejects shader models before SM 6.6, where
+; dx.op.atomicBinOp gained i64 overload support.
+
+target triple = "dxil-pc-shadermodel6.5-compute"
+
+define i64 @atomic_i64(i32 %index, i64 %value) {
+ %buffer = call target("dx.RawBuffer", i64, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index)
+ ; CHECK: Cannot create AtomicBinOp operation: Invalid overload type
+ %old = atomicrmw add ptr %ptr, i64 %value monotonic
+ ret i64 %old
+}
diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll
new file mode 100644
index 0000000000000..bc852c8a2c81f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -dxil-resource-access -dxil-op-lower %s | FileCheck %s --check-prefixes=CHECK,I32
+; RUN: opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s --check-prefixes=CHECK,I32,I64
+
+; Verify atomicrmw through a dx.resource.getpointer is lowered to
+; dx.op.atomicBinOp for UAV resources.
+
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+; CHECK-LABEL: define i32 @atomic_i32(
+define i32 @atomic_i32(i32 %index, i32 %value) {
+ %buffer = call target("dx.RawBuffer", i32, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i32, 1, 0, 0) %buffer, i32 %index)
+
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i32 %value)
+ %add = atomicrmw add ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 1, i32 %index, i32 0, i32 0, i32 %value)
+ %and = atomicrmw and ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 2, i32 %index, i32 0, i32 0, i32 %value)
+ %or = atomicrmw or ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 3, i32 %index, i32 0, i32 0, i32 %value)
+ %xor = atomicrmw xor ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 4, i32 %index, i32 0, i32 0, i32 %value)
+ %min = atomicrmw min ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 5, i32 %index, i32 0, i32 0, i32 %value)
+ %max = atomicrmw max ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 6, i32 %index, i32 0, i32 0, i32 %value)
+ %umin = atomicrmw umin ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 7, i32 %index, i32 0, i32 0, i32 %value)
+ %umax = atomicrmw umax ptr %ptr, i32 %value monotonic
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 8, i32 %index, i32 0, i32 0, i32 %value)
+ %xchg = atomicrmw xchg ptr %ptr, i32 %value monotonic
+ ret i32 %xchg
+}
+
+; CHECK-LABEL: define i32 @atomic_i32_byteaddress(
+define i32 @atomic_i32_byteaddress(i32 %offset, i32 %value) {
+ %buffer = call target("dx.RawBuffer", i8, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset)
+
+ ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %offset, i32 poison, i32 0, i32 %value)
+ %old = atomicrmw add ptr %ptr, i32 %value monotonic
+ ret i32 %old
+}
+
+; CHECK-LABEL: define i64 @atomic_i64(
+define i64 @atomic_i64(i32 %index, i64 %value) {
+ %buffer = call target("dx.RawBuffer", i64, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 2, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index)
+
+ ; I64: call i64 @dx.op.atomicBinOp.i64(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i64 %value)
+ %old = atomicrmw add ptr %ptr, i64 %value monotonic
+ ret i64 %old
+}
More information about the cfe-commits
mailing list