[clang] [llvm] [HLSL][DXIL][SPIRV] Added DeviceMemoryBarrier() and AllMemoryBarrier() intrinsics (PR #190633)
Sietze Riemersma via cfe-commits
cfe-commits at lists.llvm.org
Sun Apr 19 02:16:37 PDT 2026
https://github.com/KungFuDonkey updated https://github.com/llvm/llvm-project/pull/190633
>From 87d9da61a0f90334e08972432354f7263aa70d6f Mon Sep 17 00:00:00 2001
From: KungFuDonkey <sietze.riemersma at gmail.com>
Date: Mon, 6 Apr 2026 18:52:36 +0200
Subject: [PATCH] Added Device and AllMemoryBarriers
---
clang/include/clang/Basic/Builtins.td | 24 ++++++++++
clang/lib/CodeGen/CGHLSLBuiltins.cpp | 22 +++++++++
clang/lib/CodeGen/CGHLSLRuntime.h | 6 +++
.../lib/Headers/hlsl/hlsl_alias_intrinsics.h | 48 ++++++++++++++++++-
.../builtins/AllMemoryBarrier.hlsl | 20 ++++++++
.../AllMemoryBarrierWithGroupSync.hlsl | 20 ++++++++
.../builtins/DeviceMemoryBarrier.hlsl | 20 ++++++++
.../DeviceMemoryBarrierWithGroupSync.hlsl | 20 ++++++++
.../BuiltIns/AllMemoryBarrier-errors.hlsl | 6 +++
.../AllMemoryBarrierWithGroupSync-errors.hlsl | 6 +++
.../BuiltIns/DeviceMemoryBarrier-errors.hlsl | 6 +++
...viceMemoryBarrierWithGroupSync-errors.hlsl | 6 +++
llvm/include/llvm/IR/IntrinsicsDirectX.td | 12 +++++
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 4 ++
llvm/lib/Target/DirectX/DXIL.td | 8 ++++
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 35 +++++++++++---
.../CodeGen/DirectX/all_memory_barrier.ll | 8 ++++
.../all_memory_barrier_with_group_sync.ll | 8 ++++
.../CodeGen/DirectX/device_memory_barrier.ll | 8 ++++
.../device_memory_barrier_with_group_sync.ll | 8 ++++
.../hlsl-intrinsics/all_memory_barrier.ll | 16 +++++++
.../all_memory_barrier_with_group_sync.ll | 17 +++++++
.../hlsl-intrinsics/device_memory_barrier.ll | 16 +++++++
.../device_memory_barrier_with_group_sync.ll | 17 +++++++
.../hlsl-intrinsics/group_memory_barrier.ll | 4 +-
.../group_memory_barrier_with_group_sync.ll | 2 +-
26 files changed, 357 insertions(+), 10 deletions(-)
create mode 100644 clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl
create mode 100644 clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl
create mode 100644 clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl
create mode 100644 clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl
create mode 100644 llvm/test/CodeGen/DirectX/all_memory_barrier.ll
create mode 100644 llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll
create mode 100644 llvm/test/CodeGen/DirectX/device_memory_barrier.ll
create mode 100644 llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll
create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll
create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll
create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll
create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index b8bbc544595e2..33c176261c9e1 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5492,6 +5492,30 @@ def HLSLClip: LangBuiltin<"HLSL_LANG"> {
let Prototype = "void(...)";
}
+def HLSLAllMemoryBarrier : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_all_memory_barrier"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "void()";
+}
+
+def HLSLAllMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_all_memory_barrier_with_group_sync"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "void()";
+}
+
+def HLSLDeviceMemoryBarrier : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_device_memory_barrier"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "void()";
+}
+
+def HLSLDeviceMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_device_memory_barrier_with_group_sync"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "void()";
+}
+
def HLSLGroupMemoryBarrier : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_group_memory_barrier"];
let Attributes = [NoThrow, Const];
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index f510195dbd6cb..b82a237ecefca 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -1566,6 +1566,28 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
"clip operands types mismatch");
return handleHlslClip(E, this);
+ case Builtin::BI__builtin_hlsl_all_memory_barrier: {
+ Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic();
+ return EmitRuntimeCall(
+ Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
+ }
+ case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: {
+ Intrinsic::ID ID =
+ CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic();
+ return EmitRuntimeCall(
+ Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
+ }
+ case Builtin::BI__builtin_hlsl_device_memory_barrier: {
+ Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic();
+ return EmitRuntimeCall(
+ Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
+ }
+ case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: {
+ Intrinsic::ID ID =
+ CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic();
+ return EmitRuntimeCall(
+ Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
+ }
case Builtin::BI__builtin_hlsl_group_memory_barrier: {
Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic();
return EmitRuntimeCall(
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index b1c5b3318a11e..21e7ddf394bbd 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -189,6 +189,12 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION(NonUniformResourceIndex,
resource_nonuniformindex)
GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, resource_updatecounter)
+ GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrier, all_memory_barrier)
+ GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrierWithGroupSync,
+ all_memory_barrier_with_group_sync)
+ GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrier, device_memory_barrier)
+ GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrierWithGroupSync,
+ device_memory_barrier_with_group_sync)
GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrier, group_memory_barrier)
GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync,
group_memory_barrier_with_group_sync)
diff --git a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
index 80c415ef66644..9581c1543f468 100644
--- a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
@@ -3927,7 +3927,53 @@ _HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_radians)
float4 radians(float4);
//===----------------------------------------------------------------------===//
-// GroupMemoryBarrierbuiltins
+// AllMemoryBarrier builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn void AllMemoryBarrier(void)
+/// \brief Blocks execution of all threads in a group until all memory
+/// accesses have been completed.
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_all_memory_barrier)
+__attribute__((convergent)) void AllMemoryBarrier(void);
+
+//===----------------------------------------------------------------------===//
+// AllMemoryBarrierWithGroupSync builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn void AllMemoryBarrierWithGroupSync(void)
+/// \brief Blocks execution of all threads in a group until all memory
+/// accesses have been completed and all threads in the group have reached this
+/// call.
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_all_memory_barrier_with_group_sync)
+__attribute__((convergent)) void AllMemoryBarrierWithGroupSync(void);
+
+//===----------------------------------------------------------------------===//
+// DeviceMemoryBarrier builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn void DeviceMemoryBarrier(void)
+/// \brief Blocks execution of all threads in a group until all device memory
+/// accesses have been completed.
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_device_memory_barrier)
+__attribute__((convergent)) void DeviceMemoryBarrier(void);
+
+//===----------------------------------------------------------------------===//
+// DeviceMemoryBarrierWithGroupSync builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn void DeviceMemoryBarrierWithGroupSync(void)
+/// \brief Blocks execution of all threads in a group until all device memory
+/// accesses have been completed and all threads in the group have reached this
+/// call.
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_device_memory_barrier_with_group_sync)
+__attribute__((convergent)) void DeviceMemoryBarrierWithGroupSync(void);
+
+//===----------------------------------------------------------------------===//
+// GroupMemoryBarrier builtins
//===----------------------------------------------------------------------===//
/// \fn void GroupMemoryBarrier(void)
diff --git a/clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl
new file mode 100644
index 0000000000000..90d51c716c771
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: dxil-pc-shadermodel6.3-library %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: spirv-unknown-vulkan-compute %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV
+
+// CHECK-DXIL: define hidden void @
+// CHECK-SPIRV: define hidden spir_func void @
+void test_AllMemoryBarrier() {
+// CHECK-DXIL: call void @llvm.[[TARGET]].all.memory.barrier()
+// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].all.memory.barrier()
+ AllMemoryBarrier();
+}
+
+// CHECK: declare void @llvm.[[TARGET]].all.memory.barrier() #[[ATTRS:[0-9]+]]
+// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}}
+// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}}
diff --git a/clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl
new file mode 100644
index 0000000000000..6ddb69671e094
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: dxil-pc-shadermodel6.3-library %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: spirv-unknown-vulkan-compute %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV
+
+// CHECK-DXIL: define hidden void @
+// CHECK-SPIRV: define hidden spir_func void @
+void test_AllMemoryBarrierWithGroupSync() {
+// CHECK-DXIL: call void @llvm.[[TARGET]].all.memory.barrier.with.group.sync()
+// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].all.memory.barrier.with.group.sync()
+ AllMemoryBarrierWithGroupSync();
+}
+
+// CHECK: declare void @llvm.[[TARGET]].all.memory.barrier.with.group.sync() #[[ATTRS:[0-9]+]]
+// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}}
+// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}}
diff --git a/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl
new file mode 100644
index 0000000000000..e2c08f7775c8c
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: dxil-pc-shadermodel6.3-library %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: spirv-unknown-vulkan-compute %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV
+
+// CHECK-DXIL: define hidden void @
+// CHECK-SPIRV: define hidden spir_func void @
+void test_DeviceMemoryBarrier() {
+// CHECK-DXIL: call void @llvm.[[TARGET]].device.memory.barrier()
+// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].device.memory.barrier()
+ DeviceMemoryBarrier();
+}
+
+// CHECK: declare void @llvm.[[TARGET]].device.memory.barrier() #[[ATTRS:[0-9]+]]
+// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}}
+// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}}
diff --git a/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl
new file mode 100644
index 0000000000000..fa455f5f8338b
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: dxil-pc-shadermodel6.3-library %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN: spirv-unknown-vulkan-compute %s \
+// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV
+
+// CHECK-DXIL: define hidden void @
+// CHECK-SPIRV: define hidden spir_func void @
+void test_DeviceMemoryBarrierWithGroupSync() {
+// CHECK-DXIL: call void @llvm.[[TARGET]].device.memory.barrier.with.group.sync()
+// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].device.memory.barrier.with.group.sync()
+ DeviceMemoryBarrierWithGroupSync();
+}
+
+// CHECK: declare void @llvm.[[TARGET]].device.memory.barrier.with.group.sync() #[[ATTRS:[0-9]+]]
+// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}}
+// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}}
diff --git a/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl
new file mode 100644
index 0000000000000..63fd11a3095d8
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+void test_too_many_arg() {
+ __builtin_hlsl_all_memory_barrier(0);
+ // expected-error at -1 {{too many arguments to function call, expected 0, have 1}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl
new file mode 100644
index 0000000000000..7a50be66ebb8c
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+void test_too_many_arg() {
+ __builtin_hlsl_all_memory_barrier_with_group_sync(0);
+ // expected-error at -1 {{too many arguments to function call, expected 0, have 1}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl
new file mode 100644
index 0000000000000..16a57a36fc399
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+void test_too_many_arg() {
+ __builtin_hlsl_device_memory_barrier(0);
+ // expected-error at -1 {{too many arguments to function call, expected 0, have 1}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl
new file mode 100644
index 0000000000000..ddbd7ba768009
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+void test_too_many_arg() {
+ __builtin_hlsl_device_memory_barrier_with_group_sync(0);
+ // expected-error at -1 {{too many arguments to function call, expected 0, have 1}}
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 728bf47a17516..fecbff82e3638 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -285,6 +285,18 @@ def int_dx_firstbituhigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0,
def int_dx_firstbitshigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>;
def int_dx_firstbitlow : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>;
+def int_dx_all_memory_barrier
+ : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
+
+def int_dx_all_memory_barrier_with_group_sync
+ : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
+
+def int_dx_device_memory_barrier
+ : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
+
+def int_dx_device_memory_barrier_with_group_sync
+ : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
+
def int_dx_group_memory_barrier
: DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index d2a5fa1f08724..e0c64277ae342 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -157,6 +157,10 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]
def int_spv_quad_read_across_y : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
def int_spv_sign : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_any_ty], [IntrNoMem]>;
def int_spv_radians : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
+ def int_spv_all_memory_barrier : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
+ def int_spv_all_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
+ def int_spv_device_memory_barrier : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
+ def int_spv_device_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
def int_spv_group_memory_barrier : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
def int_spv_group_memory_barrier_with_group_sync : ClangBuiltin<"__builtin_spirv_group_barrier">,
DefaultAttrsIntrinsic<[], [], [IntrConvergent]>;
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 0a1e0114aa3bb..41f09f9db1017 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -931,6 +931,14 @@ def GetDimensions : DXILOp<72, getDimensions> {
def Barrier : DXILOp<80, barrier> {
let Doc = "inserts a memory barrier in the shader";
let intrinsics = [
+ IntrinSelect<int_dx_all_memory_barrier,
+ [IntrinArgI32<BarrierMode_AllMemoryBarrier>]>,
+ IntrinSelect<int_dx_all_memory_barrier_with_group_sync,
+ [IntrinArgI32<BarrierMode_AllMemoryBarrierWithGroupSync>]>,
+ IntrinSelect<int_dx_device_memory_barrier,
+ [IntrinArgI32<BarrierMode_DeviceMemoryBarrier>]>,
+ IntrinSelect<int_dx_device_memory_barrier_with_group_sync,
+ [IntrinArgI32<BarrierMode_DeviceMemoryBarrierWithGroupSync>]>,
IntrinSelect<int_dx_group_memory_barrier,
[IntrinArgI32<BarrierMode_GroupMemoryBarrier>]>,
IntrinSelect<int_dx_group_memory_barrier_with_group_sync,
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 9da8397efc22f..8417e591d6128 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2928,18 +2928,19 @@ bool SPIRVInstructionSelector::selectBarrierInst(MachineInstr &I,
"Device Scope must set UniformMemory and ImageMemory semantic "
"in Barrier instruction");
- Register MemSemReg = buildI32Constant(MemSem, I);
- Register ScopeReg = buildI32Constant(Scope, I);
MachineBasicBlock &BB = *I.getParent();
- auto MI =
- BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType)).addUse(ScopeReg);
+ auto MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType));
// OpControlBarrier needs to also set Execution Scope
if (WithGroupSync) {
- MI.addUse(ScopeReg);
+ Register ExecReg = buildI32Constant(SPIRV::Scope::Workgroup, I);
+ MI.addUse(ExecReg);
}
- MI.addUse(MemSemReg).constrainAllUses(TII, TRI, RBI);
+ Register ScopeReg = buildI32Constant(Scope, I);
+ Register MemSemReg = buildI32Constant(MemSem, I);
+
+ MI.addUse(ScopeReg).addUse(MemSemReg).constrainAllUses(TII, TRI, RBI);
return true;
}
@@ -4521,6 +4522,28 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
return selectFirstBitLow(ResVReg, ResType, I);
+ case Intrinsic::spv_all_memory_barrier:
+ return selectBarrierInst(I, SPIRV::Scope::Device,
+ SPIRV::MemorySemantics::UniformMemory |
+ SPIRV::MemorySemantics::ImageMemory |
+ SPIRV::MemorySemantics::WorkgroupMemory,
+ /*WithGroupSync*/ false);
+ case Intrinsic::spv_all_memory_barrier_with_group_sync:
+ return selectBarrierInst(I, SPIRV::Scope::Device,
+ SPIRV::MemorySemantics::UniformMemory |
+ SPIRV::MemorySemantics::ImageMemory |
+ SPIRV::MemorySemantics::WorkgroupMemory,
+ /*WithGroupSync*/ true);
+ case Intrinsic::spv_device_memory_barrier:
+ return selectBarrierInst(I, SPIRV::Scope::Device,
+ SPIRV::MemorySemantics::UniformMemory |
+ SPIRV::MemorySemantics::ImageMemory,
+ /*WithGroupSync*/ false);
+ case Intrinsic::spv_device_memory_barrier_with_group_sync:
+ return selectBarrierInst(I, SPIRV::Scope::Device,
+ SPIRV::MemorySemantics::UniformMemory |
+ SPIRV::MemorySemantics::ImageMemory,
+ /*WithGroupSync*/ true);
case Intrinsic::spv_group_memory_barrier:
return selectBarrierInst(I, SPIRV::Scope::Workgroup,
SPIRV::MemorySemantics::WorkgroupMemory,
diff --git a/llvm/test/CodeGen/DirectX/all_memory_barrier.ll b/llvm/test/CodeGen/DirectX/all_memory_barrier.ll
new file mode 100644
index 0000000000000..be2af03238187
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/all_memory_barrier.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
+
+define void @test_all_memory_barrier() {
+entry:
+ ; CHECK: call void @dx.op.barrier(i32 80, i32 10)
+ call void @llvm.dx.all.memory.barrier()
+ ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll
new file mode 100644
index 0000000000000..48ed1b6d36b1f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
+
+define void @test_all_memory_barrier_with_group_sync() {
+entry:
+ ; CHECK: call void @dx.op.barrier(i32 80, i32 11)
+ call void @llvm.dx.all.memory.barrier.with.group.sync()
+ ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/device_memory_barrier.ll b/llvm/test/CodeGen/DirectX/device_memory_barrier.ll
new file mode 100644
index 0000000000000..dde51af9262d0
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/device_memory_barrier.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
+
+define void @test_device_memory_barrier() {
+entry:
+ ; CHECK: call void @dx.op.barrier(i32 80, i32 2)
+ call void @llvm.dx.device.memory.barrier()
+ ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll
new file mode 100644
index 0000000000000..cb321d5cb798d
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
+
+define void @test_device_memory_barrier_with_group_sync() {
+entry:
+ ; CHECK: call void @dx.op.barrier(i32 80, i32 3)
+ call void @llvm.dx.device.memory.barrier.with.group.sync()
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll
new file mode 100644
index 0000000000000..f33724c9e6af8
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll
@@ -0,0 +1,16 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK: OpMemoryModel Logical GLSL450
+
+define void @test_all_memory_barrier() #0 {
+entry:
+ ; CHECK: %[[#TY:]] = OpTypeInt 32 0
+ ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1
+ ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2376
+ ; CHECK: OpMemoryBarrier %[[#MEM_SCOPE]] %[[#MEM_SEM]]
+ call void @llvm.spv.all.memory.barrier()
+ ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll
new file mode 100644
index 0000000000000..b39c305b45a07
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll
@@ -0,0 +1,17 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK: OpMemoryModel Logical GLSL450
+
+define void @test_all_memory_barrier_with_group_sync() #0 {
+entry:
+ ; CHECK: %[[#TY:]] = OpTypeInt 32 0
+ ; CHECK-DAG: %[[#EXEC_SCOPE:]] = OpConstant %[[#TY]] 2
+ ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1
+ ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2376
+ ; CHECK: OpControlBarrier %[[#EXEC_SCOPE]] %[[#MEM_SCOPE]] %[[#MEM_SEM]]
+ call void @llvm.spv.all.memory.barrier.with.group.sync()
+ ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll
new file mode 100644
index 0000000000000..87b9e23ded50a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll
@@ -0,0 +1,16 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK: OpMemoryModel Logical GLSL450
+
+define void @test_device_memory_barrier() #0 {
+entry:
+ ; CHECK: %[[#TY:]] = OpTypeInt 32 0
+ ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1
+ ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2120
+ ; CHECK: OpMemoryBarrier %[[#MEM_SCOPE]] %[[#MEM_SEM]]
+ call void @llvm.spv.device.memory.barrier()
+ ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll
new file mode 100644
index 0000000000000..08eb0ed5706a5
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll
@@ -0,0 +1,17 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK: OpMemoryModel Logical GLSL450
+
+define void @test_device_memory_barrier_with_group_sync() #0 {
+entry:
+ ; CHECK: %[[#TY:]] = OpTypeInt 32 0
+ ; CHECK-DAG: %[[#EXEC_SCOPE:]] = OpConstant %[[#TY]] 2
+ ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1
+ ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2120
+ ; CHECK: OpControlBarrier %[[#EXEC_SCOPE]] %[[#MEM_SCOPE]] %[[#MEM_SEM]]
+ call void @llvm.spv.device.memory.barrier.with.group.sync()
+ ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll
index 7e8ca90fb51ff..987e7b57662c9 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll
@@ -6,9 +6,9 @@
define void @test_group_memory_barrier() #0 {
entry:
; CHECK: %[[#TY:]] = OpTypeInt 32 0
+ ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 2
; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 264
- ; CHECK-DAG: %[[#EXEC_AND_MEM_SCOPE:]] = OpConstant %[[#TY]] 2
- ; CHECK: OpMemoryBarrier %[[#EXEC_AND_MEM_SCOPE]] %[[#MEM_SEM]]
+ ; CHECK: OpMemoryBarrier %[[#MEM_SCOPE]] %[[#MEM_SEM]]
call void @llvm.spv.group.memory.barrier()
ret void
}
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
index 3dcd21ef64be7..0ecd0a053ae3b 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
@@ -6,8 +6,8 @@
define void @test_group_memory_barrier_with_group_sync() #0 {
entry:
; CHECK: %[[#TY:]] = OpTypeInt 32 0
- ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 264
; CHECK-DAG: %[[#EXEC_AND_MEM_SCOPE:]] = OpConstant %[[#TY]] 2
+ ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 264
; CHECK: OpControlBarrier %[[#EXEC_AND_MEM_SCOPE]] %[[#EXEC_AND_MEM_SCOPE]] %[[#MEM_SEM]]
call void @llvm.spv.group.memory.barrier.with.group.sync()
ret void
More information about the cfe-commits
mailing list