[llvm-branch-commits] [clang] [llvm] [HLSL] Add float overload for `InterlockedExchange` (PR #222163)
Joshua Batista via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 8 14:50:29 PDT 2026
https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/222163
This PR adds the float overload of `InterlockedExchange`, and the
`InterlockedExchangeFloat` method on `RWByteAddressBuffer` and
`RasterizerOrderedByteAddressBuffer`. A `ByteAddressBuffer` carries no element
type, so the method name states the type, as DXC does.
`atomicrmw xchg` accepts a float operand, so Clang emits it directly. The
`DXILLegalizePass` then bitcasts the operand to an integer, because DXIL has
no float atomic exchange. The operation reuses the 32-bit integer DXIL
operation, so it needs no capability bits and works from shader model 6.0. A
test checks both halves of that: the float form is accepted at SM 6.0, and the
64-bit form is still rejected there.
### Why the builtin gains `CustomTypeChecking`
The HLSL interlocked builtins are declared with a variadic type in
`Builtins.td`, so Clang applies the default argument promotions to their
arguments. That promotion turns a `float` argument into a `double` before Sema
sees it. The Sema check then rejects the call and names the type as `'double'`
in the diagnostic, which does not match what the user wrote.
`CustomTypeChecking` suppresses that promotion, so the argument keeps its
`float` type and the diagnostic names it correctly.
Only the builtins that can see a float argument need the attribute. This PR
sets it on `__builtin_hlsl_interlocked_exchange` alone. The later float
builtins in this stack set it as they are added. The integer-only builtins do
not set it, because they never see a float and the promotion cannot affect
them.
Fixes: https://github.com/llvm/llvm-project/issues/99129
Assisted by: Github Copilot
>From c888d2195ee2946ad75188b738ca42b940dffd1a Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Fri, 4 Sep 2026 14:54:47 -0700
Subject: [PATCH] First attempt implementing float InterlockedExchange
---
clang/include/clang/Basic/Builtins.td | 5 ++-
.../clang/Basic/DiagnosticSemaKinds.td | 2 +-
clang/lib/CodeGen/CGHLSLBuiltins.cpp | 7 +++-
clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 6 +++
clang/lib/Sema/HLSLExternalSemaSource.cpp | 14 +++++--
clang/lib/Sema/SemaHLSL.cpp | 12 ++++--
.../builtins/InterlockedExchange.hlsl | 11 +++++
...ByteAddressBuffer-InterlockedExchange.hlsl | 16 ++++++++
...sBuffer-InterlockedExchangeFloat-sm60.hlsl | 40 +++++++++++++++++++
.../BuiltIns/InterlockedExchange-errors.hlsl | 35 +++++++++-------
llvm/lib/Target/DirectX/DXILLegalizePass.cpp | 29 ++++++++++++++
.../lib/Target/DirectX/DXILResourceAccess.cpp | 20 ++++++++--
.../DirectX/LegalizeAtomicExchangeFloat.ll | 39 ++++++++++++++++++
.../DirectX/ResourceAtomicExchangeFloat.ll | 38 ++++++++++++++++++
14 files changed, 246 insertions(+), 28 deletions(-)
create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedExchangeFloat-sm60.hlsl
create mode 100644 llvm/test/CodeGen/DirectX/LegalizeAtomicExchangeFloat.ll
create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicExchangeFloat.ll
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 41426e9784fc1..e9a8a73afbd3c 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5559,7 +5559,10 @@ def HLSLInterlockedAnd : LangBuiltin<"HLSL_LANG"> {
def HLSLInterlockedExchange : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_interlocked_exchange"];
- let Attributes = [NoThrow];
+ // SemaHLSL checks these arguments itself. Custom type checking also stops
+ // the default variadic promotion, which would otherwise widen a float
+ // `value` to double when a ByteAddressBuffer method calls this builtin.
+ let Attributes = [NoThrow, CustomTypeChecking];
let Prototype = "void (...)";
}
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 6c4339b6175eb..d03c2cad88a0e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -13375,7 +13375,7 @@ def err_builtin_invalid_arg_type: Error<
// An 'or' if non-empty second and third components are combined
"%plural{0:|:%plural{0:|:or }2}3"
// Third component: floating-point types
- "%select{|floating-point|16 or 32 bit floating-point}3"
+ "%select{|floating-point|16 or 32 bit floating-point|32 bit floating-point}3"
// A space after a non-empty third component
"%plural{0:|: }3"
"%plural{[0,3]:type|:types}1 (was %4)">;
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index 2d331553c2295..d6c501a20f900 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -317,8 +317,11 @@ static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E,
LValue DestLV = CGF.EmitLValue(E->getArg(0));
Address DestAddr = DestLV.getAddress();
Value *Val = CGF.EmitScalarExpr(E->getArg(1));
- assert(E->getArg(1)->getType()->isIntegerType() &&
- "Intrinsic InterlockedOp value operand must be an integer");
+ assert((E->getArg(1)->getType()->isIntegerType() ||
+ (Op == llvm::AtomicRMWInst::Xchg &&
+ E->getArg(1)->getType()->isFloatingType())) &&
+ "Intrinsic InterlockedOp value operand must be an integer, or a "
+ "float for InterlockedExchange");
// Scopeless atomics will default to CrossDevice, which is illegal in Vulkan.
// Set the memory scope: Workgroup for groupshared, otherwise Device.
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
index c29756e9135c8..63f1dbb518edd 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
@@ -1756,6 +1756,12 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() {
addByteAddressBufferInterlockedMethod(
"InterlockedExchange", AST.UnsignedIntTy,
"__builtin_hlsl_interlocked_exchange", /*RequiresOriginalValue=*/true);
+ // The float exchange reuses the 32-bit integer DXIL operation, so it needs
+ // no capability bits and works from SM 6.0. ByteAddressBuffer carries no
+ // element type, so the method name states the type.
+ addByteAddressBufferInterlockedMethod("InterlockedExchangeFloat", AST.FloatTy,
+ "__builtin_hlsl_interlocked_exchange",
+ /*RequiresOriginalValue=*/true);
addByteAddressBufferInterlockedMethod("InterlockedMax", AST.IntTy,
"__builtin_hlsl_interlocked_max");
addByteAddressBufferInterlockedMethod("InterlockedMax", AST.UnsignedIntTy,
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index 983273cb205eb..b1c64f99629b2 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -867,13 +867,18 @@ static void buildAtomicOverload(Sema &S, NamespaceDecl *NS, StringRef FuncName,
// Synthesize the InterlockedFunc overload set: {int, uint, int64_t, uint64_t}
// x {groupshared, device} x {2-arg, 3-arg}. Operations that always report the
// previous value, such as InterlockedExchange, only get the 3-arg form.
+// InterlockedExchange also accepts float, which lowers to a bitwise exchange
+// of the 32-bit pattern.
static void defineHLSLInterlockedFunc(Sema &S, NamespaceDecl *NS,
StringRef FuncName, StringRef BuiltinName,
- bool RequiresOriginalValue = false) {
+ bool RequiresOriginalValue = false,
+ bool SupportsFloat = false) {
ASTContext &AST = S.getASTContext();
// HLSL: int64_t == long, uint64_t == unsigned long (see hlsl_basic_types.h).
- QualType Elems[] = {AST.IntTy, AST.UnsignedIntTy, AST.LongTy,
- AST.UnsignedLongTy};
+ SmallVector<QualType, 5> Elems = {AST.IntTy, AST.UnsignedIntTy, AST.LongTy,
+ AST.UnsignedLongTy};
+ if (SupportsFloat)
+ Elems.push_back(AST.FloatTy);
LangAS AddrSpaces[] = {LangAS::hlsl_groupshared, LangAS::hlsl_device};
for (QualType ElemTy : Elems)
@@ -892,7 +897,8 @@ void HLSLExternalSemaSource::defineHLSLAtomicIntrinsics() {
"__builtin_hlsl_interlocked_and");
defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedExchange",
"__builtin_hlsl_interlocked_exchange",
- /*RequiresOriginalValue=*/true);
+ /*RequiresOriginalValue=*/true,
+ /*SupportsFloat=*/true);
defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedMax",
"__builtin_hlsl_interlocked_max");
defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedMin",
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 883140d9dd8fb..50735771f52a3 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -4742,11 +4742,17 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
}
QualType DestTy = TheCall->getArg(0)->getType().getUnqualifiedType();
- if (!DestTy->isIntegerType()) {
+ // InterlockedExchange also operates on float. DXIL lowers that as a
+ // bitwise exchange of the value's bit pattern, and DXC accepts 32-bit
+ // float only, so half and double are rejected.
+ const bool AllowsFloat =
+ BuiltinID == Builtin::BI__builtin_hlsl_interlocked_exchange;
+ if (!DestTy->isIntegerType() &&
+ !(AllowsFloat && DestTy->isSpecificBuiltinType(BuiltinType::Float))) {
SemaRef.Diag(TheCall->getArg(0)->getBeginLoc(),
diag::err_builtin_invalid_arg_type)
- << /*ordinal=*/1 << /*scalar*/ 1 << /*integer*/ 1 << /*no float*/ 0
- << DestTy;
+ << /*ordinal=*/1 << /*scalar*/ 1 << /*integer*/ 1
+ << /*32 bit floating-point*/ (AllowsFloat ? 3 : 0) << DestTy;
return true;
}
diff --git a/clang/test/CodeGenHLSL/builtins/InterlockedExchange.hlsl b/clang/test/CodeGenHLSL/builtins/InterlockedExchange.hlsl
index d4a27fdd1540e..ad275dfa4df77 100644
--- a/clang/test/CodeGenHLSL/builtins/InterlockedExchange.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/InterlockedExchange.hlsl
@@ -14,6 +14,7 @@ groupshared int gs_i32;
groupshared uint gs_u32;
groupshared int64_t gs_i64;
groupshared uint64_t gs_u64;
+groupshared float gs_f32;
// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int_3arg
// DXCHECK: %[[R:.*]] = atomicrmw xchg ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic
@@ -46,3 +47,13 @@ export void test_int64_3arg(int64_t v, out int64_t orig) {
export void test_uint64_3arg(uint64_t v, out uint64_t orig) {
InterlockedExchange(gs_u64, v, orig);
}
+
+// The float overload keeps the float type in the IR. DXIL converts it to an
+// i32 exchange later, and SPIR-V selects OpAtomicExchange directly.
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_float_3arg
+// DXCHECK: %[[R:.*]] = atomicrmw xchg ptr addrspace(3) {{.*}}@gs_f32{{.*}}, float %{{.*}} syncscope("workgroup") monotonic
+// SPVCHECK: %[[R:.*]] = atomicrmw xchg ptr addrspace(3) {{.*}}@gs_f32{{.*}}, float %{{.*}} syncscope("workgroup") monotonic
+// CHECK: store float %[[R]], ptr {{.*}}
+export void test_float_3arg(float v, out float orig) {
+ InterlockedExchange(gs_f32, v, orig);
+}
diff --git a/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedExchange.hlsl b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedExchange.hlsl
index fc775ef7a33b1..b9960905a125a 100644
--- a/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedExchange.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedExchange.hlsl
@@ -38,3 +38,19 @@ export void test_bab_uint_3arg(uint off, uint v, out uint orig) {
export void test_bab_uint64_3arg(uint off, uint64_t v, out uint64_t orig) {
BAB.InterlockedExchange64(off, v, orig);
}
+
+// ByteAddressBuffer holds no element type, so the float method carries the
+// type in its name. The value keeps the float type here; DXIL converts it to
+// an i32 exchange later.
+// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_float_3arg
+// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}}
+// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}})
+// DXCHECK: %[[R:.*]] = atomicrmw xchg ptr %[[PTR]], float %{{.*}} syncscope("device") monotonic
+// DXCHECK: store float %[[R]], ptr {{.*}}
+// SPVCHECK: %[[HANDLE:.*]] = load target("spirv.VulkanBuffer", [0 x i8], 12, 1), ptr {{.*}}
+// SPVCHECK: %[[PTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i8_12_1t.i32(target("spirv.VulkanBuffer", [0 x i8], 12, 1) %[[HANDLE]], i32 %{{.*}})
+// SPVCHECK: %[[R:.*]] = atomicrmw xchg ptr addrspace(11) %[[PTR]], float %{{.*}} syncscope("device") monotonic
+// SPVCHECK: store float %[[R]], ptr {{.*}}
+export void test_bab_float_3arg(uint off, float v, out float orig) {
+ BAB.InterlockedExchangeFloat(off, v, orig);
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedExchangeFloat-sm60.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedExchangeFloat-sm60.hlsl
new file mode 100644
index 0000000000000..c4b1f34455f14
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedExchangeFloat-sm60.hlsl
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.0-library %s -fsyntax-only -verify \
+// RUN: -verify-ignore-unexpected=warning
+
+// The float exchange reuses the 32-bit integer DXIL operation, so it needs no
+// capability bits and works from SM 6.0. The 64-bit exchange needs SM 6.6.
+// This file checks both halves, so it proves the two are gated differently.
+
+RWByteAddressBuffer BAB : register(u0);
+RasterizerOrderedByteAddressBuffer ROVB : register(u1);
+groupshared float gs_f32;
+groupshared int64_t gs_i64;
+
+void sm60_bab_float_ok(uint off, float v, out float orig) {
+ BAB.InterlockedExchangeFloat(off, v, orig);
+}
+
+void sm60_rovb_float_ok(uint off, float v, out float orig) {
+ ROVB.InterlockedExchangeFloat(off, v, orig);
+}
+
+void sm60_free_function_ok(float v) {
+ float orig;
+ InterlockedExchange(gs_f32, v, orig);
+}
+
+void sm60_direct_builtin_ok(float v) {
+ float orig;
+ __builtin_hlsl_interlocked_exchange(gs_f32, v, orig);
+}
+
+void sm60_no_bab_exchange64(uint off, uint64_t v, out uint64_t orig) {
+ BAB.InterlockedExchange64(off, v, orig);
+ // expected-error at -1 {{no member named 'InterlockedExchange64' in 'hlsl::RWByteAddressBuffer'}}
+}
+
+void sm60_no_direct_builtin_i64(int64_t v, out int64_t orig) {
+ __builtin_hlsl_interlocked_exchange(gs_i64, v, orig);
+ // expected-error at -1 {{'__builtin_hlsl_interlocked_exchange' requires shader model 6.6 or newer}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/InterlockedExchange-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/InterlockedExchange-errors.hlsl
index bf6a1757b42f2..3d70256bcddef 100644
--- a/clang/test/SemaHLSL/BuiltIns/InterlockedExchange-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/InterlockedExchange-errors.hlsl
@@ -3,53 +3,54 @@
// RUN: -disable-llvm-passes -verify
// InterlockedExchange is provided as a set of address-space-qualified
-// overloads (groupshared/device, {int,uint,int64_t,uint64_t}). It always
-// reports the previous value, so there is no 2-argument form.
+// overloads (groupshared/device, {int,uint,int64_t,uint64_t,float}). It always
+// reports the previous value, so there is no 2-argument form. Only 32-bit
+// float is accepted, so double has no overload.
groupshared int gs_i32;
-groupshared float gs_f32;
+groupshared double gs_f64;
struct S { int x; };
groupshared S gs_s;
void too_few() {
InterlockedExchange(gs_i32); // expected-error{{no matching function for call to 'InterlockedExchange'}}
- // expected-note@*:* 8 {{candidate function}}
+ // expected-note@*:* 10 {{candidate function}}
}
void missing_original_value(int v) {
InterlockedExchange(gs_i32, v); // expected-error{{no matching function for call to 'InterlockedExchange'}}
- // expected-note@*:* 8 {{candidate function}}
+ // expected-note@*:* 10 {{candidate function}}
}
void too_many(int v, int extra) {
int orig;
InterlockedExchange(gs_i32, v, orig, extra); // expected-error{{no matching function for call to 'InterlockedExchange'}}
- // expected-note@*:* 8 {{candidate function}}
+ // expected-note@*:* 10 {{candidate function}}
}
void local_dest(int v) {
int dest;
int orig;
InterlockedExchange(dest, v, orig); // expected-error{{no matching function for call to 'InterlockedExchange'}}
- // expected-note@*:* 8 {{candidate function}}
+ // expected-note@*:* 10 {{candidate function}}
}
-void float_dest(float v) {
- float orig;
- InterlockedExchange(gs_f32, v, orig); // expected-error{{no matching function for call to 'InterlockedExchange'}}
- // expected-note@*:* 8 {{candidate function}}
+void double_dest(double v) {
+ double orig;
+ InterlockedExchange(gs_f64, v, orig); // expected-error{{no matching function for call to 'InterlockedExchange'}}
+ // expected-note@*:* 10 {{candidate function}}
}
void struct_dest(int v) {
int orig;
InterlockedExchange(gs_s, v, orig); // expected-error{{no matching function for call to 'InterlockedExchange'}}
- // expected-note@*:* 8 {{candidate function}}
+ // expected-note@*:* 10 {{candidate function}}
}
void mismatched_orig_type(int v) {
uint orig;
InterlockedExchange(gs_i32, v, orig); // expected-error{{no matching function for call to 'InterlockedExchange'}}
- // expected-note@*:* 8 {{candidate function}}
+ // expected-note@*:* 10 {{candidate function}}
}
void direct_too_few() {
@@ -72,7 +73,13 @@ void direct_non_integer_dest() {
S local_s;
S orig;
__builtin_hlsl_interlocked_exchange(local_s, 1, orig);
- // expected-error at -1 {{1st argument must be a scalar integer type (was 'S')}}
+ // expected-error at -1 {{1st argument must be a scalar integer or 32 bit floating-point type (was 'S')}}
+}
+
+void direct_double_dest(double v) {
+ double orig;
+ __builtin_hlsl_interlocked_exchange(gs_f64, v, orig);
+ // expected-error at -1 {{1st argument must be a scalar integer or 32 bit floating-point type (was 'double')}}
}
void direct_nonlvalue_dest(int v) {
diff --git a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
index b7c2083753f89..0149d8baf028a 100644
--- a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
+++ b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
@@ -356,6 +356,34 @@ static bool updateFnegToFsub(Instruction &I,
return true;
}
+// DXIL has no floating-point atomic operation. A float exchange only moves the
+// bit pattern, so exchange an integer of the same width instead. Opaque
+// pointers keep the pointer operand type-agnostic, so only the value and the
+// result need a cast. This matches what DXC emits for groupshared memory.
+static bool
+legalizeFloatAtomicExchange(Instruction &I,
+ SmallVectorImpl<Instruction *> &ToRemove,
+ DenseMap<Value *, Value *> &) {
+ auto *AI = dyn_cast<AtomicRMWInst>(&I);
+ if (!AI || AI->getOperation() != AtomicRMWInst::Xchg)
+ return false;
+
+ Type *ValTy = AI->getValOperand()->getType();
+ if (!ValTy->isFloatingPointTy())
+ return false;
+
+ IRBuilder<> Builder(AI);
+ Type *IntTy = Builder.getIntNTy(ValTy->getPrimitiveSizeInBits());
+ Value *Val = Builder.CreateBitCast(AI->getValOperand(), IntTy);
+ AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
+ AtomicRMWInst::Xchg, AI->getPointerOperand(), Val, AI->getAlign(),
+ AI->getOrdering(), AI->getSyncScopeID());
+ NewAI->setVolatile(AI->isVolatile());
+ AI->replaceAllUsesWith(Builder.CreateBitCast(NewAI, ValTy));
+ ToRemove.push_back(AI);
+ return true;
+}
+
static bool
legalizeGetHighLowi64Bytes(Instruction &I,
SmallVectorImpl<Instruction *> &ToRemove,
@@ -548,6 +576,7 @@ class DXILLegalizationPipeline {
LegalizationPipeline[Stage1].push_back(legalizeGetHighLowi64Bytes);
LegalizationPipeline[Stage1].push_back(legalizeFreeze);
LegalizationPipeline[Stage1].push_back(updateFnegToFsub);
+ LegalizationPipeline[Stage1].push_back(legalizeFloatAtomicExchange);
// Note: legalizeGetHighLowi64Bytes and
// downcastI64toI32InsertExtractElements both modify extractelement, so they
// must run staggered stages. legalizeGetHighLowi64Bytes runs first b\c it
diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
index e8ac9bfb0c99e..b70a45e5abad4 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp
@@ -370,12 +370,26 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
Value *BinOp = Builder.getInt32(static_cast<uint32_t>(*BinOpCode));
+ // DXIL has no floating-point atomic op. A float exchange only moves the bit
+ // pattern, so cast the value to an integer of the same width, exchange, and
+ // cast the original value back. This matches what DXC emits.
+ Value *Val = AI->getValOperand();
+ Type *ValTy = Val->getType();
+ Type *OpTy = ValTy;
+ if (ValTy->isFloatingPointTy()) {
+ OpTy = Builder.getIntNTy(ValTy->getPrimitiveSizeInBits());
+ Val = Builder.CreateBitCast(Val, OpTy);
+ }
+
// Emit the target-independent intrinsic; DXILOpLowering lowers it to the
// DXIL `AtomicBinOp` op and handles the target-ext-typed handle cast via
// its `createTmpHandleCast` bookkeeping.
- Value *Result = Builder.CreateIntrinsic(
- AI->getType(), Intrinsic::dx_resource_atomic_binop,
- {II->getOperand(0), BinOp, Index, Offset, AI->getValOperand()});
+ Value *Result =
+ Builder.CreateIntrinsic(OpTy, Intrinsic::dx_resource_atomic_binop,
+ {II->getOperand(0), BinOp, Index, Offset, Val});
+
+ if (OpTy != ValTy)
+ Result = Builder.CreateBitCast(Result, ValTy);
AI->replaceAllUsesWith(Result);
}
diff --git a/llvm/test/CodeGen/DirectX/LegalizeAtomicExchangeFloat.ll b/llvm/test/CodeGen/DirectX/LegalizeAtomicExchangeFloat.ll
new file mode 100644
index 0000000000000..8becba6996441
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/LegalizeAtomicExchangeFloat.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -dxil-legalize -mtriple=dxil-pc-shadermodel6.0-compute %s | FileCheck %s
+
+; DXIL has no floating-point atomic op. A float exchange on groupshared memory
+; only moves the bit pattern, so it becomes an i32 exchange with a bitcast on
+; the value and on the result. Opaque pointers leave the pointer operand
+; unchanged.
+
+target triple = "dxil-pc-shadermodel6.0-compute"
+
+ at gs = external addrspace(3) global float
+
+; CHECK-LABEL: define float @gs_xchg_float
+define float @gs_xchg_float(float %val) {
+ ; CHECK: [[CAST:%.*]] = bitcast float %val to i32
+ ; CHECK: [[OLD:%.*]] = atomicrmw xchg ptr addrspace(3) @gs, i32 [[CAST]] syncscope("workgroup") monotonic
+ ; CHECK: [[RES:%.*]] = bitcast i32 [[OLD]] to float
+ %old = atomicrmw xchg ptr addrspace(3) @gs, float %val syncscope("workgroup") monotonic
+ ; CHECK: ret float [[RES]]
+ ret float %old
+}
+
+; An integer exchange must pass through with no bitcast.
+; CHECK-LABEL: define i32 @gs_xchg_i32
+define i32 @gs_xchg_i32(ptr addrspace(3) %p, i32 %val) {
+ ; CHECK-NOT: bitcast
+ ; CHECK: atomicrmw xchg ptr addrspace(3) %p, i32 %val syncscope("workgroup") monotonic
+ %old = atomicrmw xchg ptr addrspace(3) %p, i32 %val syncscope("workgroup") monotonic
+ ret i32 %old
+}
+
+; Only exchange is rewritten. DXIL supports a native float atomic add nowhere,
+; but fadd must not be silently turned into an integer add.
+; CHECK-LABEL: define float @gs_fadd_float
+define float @gs_fadd_float(ptr addrspace(3) %p, float %val) {
+ ; CHECK-NOT: bitcast
+ ; CHECK: atomicrmw fadd ptr addrspace(3) %p, float %val syncscope("workgroup") monotonic
+ %old = atomicrmw fadd ptr addrspace(3) %p, float %val syncscope("workgroup") monotonic
+ ret float %old
+}
diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicExchangeFloat.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicExchangeFloat.ll
new file mode 100644
index 0000000000000..e0418b5c33348
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ResourceAtomicExchangeFloat.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.0-compute %s | FileCheck %s
+
+; DXIL has no floating-point atomic op. A float exchange only moves the bit
+; pattern, so it lowers to an i32 AtomicBinOp with a bitcast on the value and
+; on the returned original value. This needs no capability bits, so it works
+; from SM 6.0.
+
+target triple = "dxil-pc-shadermodel6.0-compute"
+
+; CHECK-LABEL: define float @bab_xchg_float
+define float @bab_xchg_float(i32 %offset, float %val) {
+ %buffer = call target("dx.RawBuffer", i8, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset)
+ ; CHECK: [[CAST:%.*]] = bitcast float %val to i32
+ ; CHECK: [[OLD:%.*]] = call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 8, i32 %offset, i32 poison, i32 0, i32 [[CAST]])
+ ; CHECK: [[RES:%.*]] = bitcast i32 [[OLD]] to float
+ %old = atomicrmw xchg ptr %ptr, float %val monotonic
+ ; CHECK: ret float [[RES]]
+ ret float %old
+}
+
+; A StructuredBuffer of float keeps the struct index in coord0 and the byte
+; offset in coord1.
+; CHECK-LABEL: define float @sbuf_xchg_float
+define float @sbuf_xchg_float(i32 %index, float %val) {
+ %buffer = call target("dx.RawBuffer", float, 1, 0, 0)
+ @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null)
+ %ptr = call ptr @llvm.dx.resource.getpointer(
+ target("dx.RawBuffer", float, 1, 0, 0) %buffer, i32 %index)
+ ; CHECK: [[CAST:%.*]] = bitcast float %val to i32
+ ; CHECK: [[OLD:%.*]] = call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 8, i32 %index, i32 0, i32 0, i32 [[CAST]])
+ ; CHECK: [[RES:%.*]] = bitcast i32 [[OLD]] to float
+ %old = atomicrmw xchg ptr %ptr, float %val monotonic
+ ; CHECK: ret float [[RES]]
+ ret float %old
+}
More information about the llvm-branch-commits
mailing list