[llvm-branch-commits] [clang] [llvm] [HLSL] Add WaveMultiPrefixCountBits (PR #238132)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 2 15:00:11 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-directx
Author: Joshua Batista (bob80905)
<details>
<summary>Changes</summary>
This is the third PR in the WaveMultiPrefix stack. It builds on #<!-- -->238097.
This PR adds Clang and DXIL support for `WaveMultiPrefixCountBits(bool value, uint4 mask)` in Shader Model 6.5 or newer. It returns a `uint` and lowers to DXIL opcode 167, `WaveMultiPrefixBitCount`.
This PR does not add SPIR-V support. SPIR-V calls report that the builtin is not supported on the target.
Fixes #<!-- -->99211.
---
Full diff: https://github.com/llvm/llvm-project/pull/238132.diff
12 Files Affected:
- (modified) clang/include/clang/Basic/Builtins.td (+6)
- (modified) clang/include/clang/Basic/HLSLIntrinsics.td (+9)
- (modified) clang/lib/CodeGen/CGHLSLBuiltins.cpp (+4)
- (modified) clang/lib/Sema/SemaHLSL.cpp (+32-10)
- (added) clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl (+45)
- (added) clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl (+63)
- (modified) llvm/include/llvm/IR/IntrinsicsDirectX.td (+6)
- (modified) llvm/lib/Target/DirectX/DXIL.td (+9)
- (modified) llvm/lib/Target/DirectX/DXILShaderFlags.cpp (+1)
- (modified) llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll (+7)
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll (+20)
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll (+9)
``````````diff
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index ab4b7f4b1187a..e2679df195475 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5801,6 +5801,12 @@ def HLSLWaveMultiPrefixBitXor : LangBuiltin<"HLSL_LANG"> {
let Prototype = "void(...)";
}
+def HLSLWaveMultiPrefixCountBits : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_wave_multi_prefix_count_bits"];
+ let Attributes = [NoThrow, Const, CustomTypeChecking];
+ let Prototype = "unsigned int(bool, _ExtVector<4, unsigned int>)";
+}
+
def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_wave_prefix_product"];
let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td
index 300d46a7fd324..89728b4ba885c 100644
--- a/clang/include/clang/Basic/HLSLIntrinsics.td
+++ b/clang/include/clang/Basic/HLSLIntrinsics.td
@@ -1903,6 +1903,15 @@ def hlsl_wave_multi_prefix_bit_xor :
let Availability = SM6_5;
}
+def hlsl_wave_multi_prefix_count_bits :
+ HLSLBuiltin<"WaveMultiPrefixCountBits",
+ "__builtin_hlsl_wave_multi_prefix_count_bits"> {
+ let Args = [BoolTy, VectorType<UIntTy, 4>];
+ let ReturnType = UIntTy;
+ let IsConvergent = 1;
+ let Availability = SM6_5;
+}
+
// Returns the count of bits set to 1 on prior lanes.
def hlsl_wave_prefix_count_bits :
HLSLBuiltin<"WavePrefixCountBits",
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index 3afc8a0668f27..229a1cc4f830a 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -1721,6 +1721,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
return EmitIntrinsicCall(IID, {OpExpr->getType()}, ArrayRef{OpExpr},
"hlsl.wave.prefix.sum");
}
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits:
case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: {
Value *ValueOp = EmitScalarExpr(E->getArg(0));
@@ -1728,6 +1729,9 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
SmallVector<Value *, 5> Args{ValueOp};
for (unsigned I = 0; I != 4; ++I)
Args.push_back(Builder.CreateExtractElement(Mask, I));
+ if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits)
+ return EmitIntrinsicCall(Intrinsic::dx_wave_multi_prefix_bit_count, Args,
+ "hlsl.wave.multi.prefix.bit.count");
if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and)
return EmitIntrinsicCall(
CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index d0da34ef79f89..e0a92ff1d472c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -4012,6 +4012,18 @@ static bool CheckWavePrefix(Sema *S, CallExpr *TheCall) {
return false;
}
+static bool CheckWaveMultiPrefixMask(Sema *S, CallExpr *TheCall) {
+ QualType MaskTy = TheCall->getArg(1)->getType();
+ QualType UInt4Ty = S->Context.getExtVectorType(S->Context.UnsignedIntTy, 4);
+ if (S->Context.hasSameUnqualifiedType(MaskTy, UInt4Ty))
+ return false;
+
+ S->Diag(TheCall->getArg(1)->getBeginLoc(),
+ diag::err_typecheck_convert_incompatible)
+ << MaskTy << UInt4Ty << 1 << 0 << 0;
+ return true;
+}
+
static bool CheckBoolSelect(Sema *S, CallExpr *TheCall) {
assert(TheCall->getNumArgs() == 3);
Expr *Arg1 = TheCall->getArg(1);
@@ -5302,8 +5314,11 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
return true;
break;
}
- case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
- if (SemaRef.checkArgCount(TheCall, 1))
+ case Builtin::BI__builtin_hlsl_wave_prefix_count_bits:
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits: {
+ bool IsMultiPrefix =
+ BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits;
+ if (SemaRef.checkArgCount(TheCall, IsMultiPrefix ? 2 : 1))
return true;
QualType ArgType = TheCall->getArg(0)->getType();
@@ -5322,6 +5337,20 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
return true;
}
+ if (IsMultiPrefix) {
+ if (CheckWaveMultiPrefixMask(&SemaRef, TheCall))
+ return true;
+
+ // Alias validation uses calls with an invalid source location.
+ if (TheCall->getBeginLoc().isValid() &&
+ !getASTContext().getTargetInfo().getTriple().isDXIL()) {
+ SemaRef.Diag(TheCall->getBeginLoc(),
+ diag::err_builtin_target_unsupported)
+ << TheCall->getSourceRange();
+ return true;
+ }
+ }
+
break;
}
case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
@@ -5395,15 +5424,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
return true;
}
- QualType MaskTy = TheCall->getArg(1)->getType();
- QualType UInt4Ty =
- getASTContext().getExtVectorType(getASTContext().UnsignedIntTy, 4);
- if (!getASTContext().hasSameUnqualifiedType(MaskTy, UInt4Ty)) {
- SemaRef.Diag(TheCall->getArg(1)->getBeginLoc(),
- diag::err_typecheck_convert_incompatible)
- << MaskTy << UInt4Ty << 1 << 0 << 0;
+ if (CheckWaveMultiPrefixMask(&SemaRef, TheCall))
return true;
- }
TheCall->setType(ValueTy);
break;
diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl
new file mode 100644
index 0000000000000..4b7c8445d5c7e
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl
@@ -0,0 +1,45 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | FileCheck %s
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | \
+// RUN: opt -S -strip-convergence-intrinsics -dxil-op-lower | \
+// RUN: FileCheck %s --check-prefix=DXIL
+
+// CHECK-LABEL: test_bool
+// DXIL-LABEL: test_bool
+uint test_bool(bool value, uint4 mask) {
+ // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry()
+ // CHECK: %[[VALUE:.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: %[[MASK:.*]] = load <4 x i32>
+ // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0
+ // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1
+ // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2
+ // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3
+ // CHECK: %[[RET:.*]] = call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 %[[VALUE]], i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ]
+ // CHECK: ret i32 %[[RET]]
+ // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+ return WaveMultiPrefixCountBits(value, mask);
+}
+
+// CHECK: declare i32 @llvm.dx.wave.multi.prefix.bit.count(i1, i32, i32, i32, i32) #[[ATTR:[0-9]+]]
+
+// CHECK-LABEL: test_true
+// DXIL-LABEL: test_true
+uint test_true() {
+ // CHECK: call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 true, i32 1, i32 2, i32 4, i32 8)
+ // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 true, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixCountBits(true, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_false
+// DXIL-LABEL: test_false
+uint test_false() {
+ // CHECK: call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 false, i32 16, i32 32, i32 64, i32 128)
+ // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 false, i32 16, i32 32, i32 64, i32 128)
+ return __builtin_hlsl_wave_multi_prefix_count_bits(
+ false, uint4(16, 32, 64, 128));
+}
+
+// CHECK: attributes #[[ATTR]] = { convergent {{.*}}memory(none) }
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl
new file mode 100644
index 0000000000000..4b0944d349395
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \
+// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_UNSUPPORTED \
+// RUN: -triple spirv-pc-vulkan-library -verify=spirv %s
+
+#ifdef TEST_AVAILABILITY
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main() {
+ (void)WaveMultiPrefixCountBits(true, uint4(1, 2, 4, 8));
+ // availability-error at -1 {{only available on Shader Model 6.5 or newer}}
+ // availability-note@*:* {{introduced in Shader Model 6.5}}
+}
+#elif defined(TEST_UNSUPPORTED)
+uint test_public(bool value, uint4 mask) {
+ return WaveMultiPrefixCountBits(value, mask);
+ // spirv-error at -1 {{builtin is not supported on this target}}
+}
+
+uint test_builtin(bool value, uint4 mask) {
+ return __builtin_hlsl_wave_multi_prefix_count_bits(value, mask);
+ // spirv-error at -1 {{builtin is not supported on this target}}
+}
+#else
+void test_arg_count(bool value, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_count_bits(value);
+ // expected-error at -1 {{too few arguments to function call, expected 2}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(value, mask, mask);
+ // expected-error at -1 {{too many arguments to function call, expected 2}}
+}
+
+struct S { uint value; };
+
+void test_value_types(float f, int i, bool2 bv, S s, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_count_bits(f, mask);
+ // expected-error at -1 {{invalid operand of type 'float'}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(i, mask);
+ // expected-error at -1 {{invalid operand of type 'int'}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(bv, mask);
+ // expected-error at -1 {{invalid operand of type 'bool2'}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(s, mask);
+ // expected-error at -1 {{invalid operand of type 'S'}}
+}
+
+void test_mask_types(bool value, uint scalar, uint3 short_mask,
+ int4 signed_mask, float4 float_mask,
+ uint64_t4 wide_mask, S struct_mask) {
+ __builtin_hlsl_wave_multi_prefix_count_bits(value, scalar);
+ // expected-error at -1 {{passing 'uint' (aka 'unsigned int')}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(value, short_mask);
+ // expected-error at -1 {{passing 'uint3' (aka 'vector<uint, 3>')}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(value, signed_mask);
+ // expected-error at -1 {{passing 'int4' (aka 'vector<int, 4>')}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(value, float_mask);
+ // expected-error at -1 {{passing 'float4' (aka 'vector<float, 4>')}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(value, wide_mask);
+ // expected-error at -1 {{passing 'uint64_t4'}}
+ __builtin_hlsl_wave_multi_prefix_count_bits(value, struct_mask);
+ // expected-error at -1 {{passing 'S' to parameter of incompatible type}}
+}
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index c292a0a46b2ae..6343cd2332bd2 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -488,6 +488,12 @@ let TargetPrefix = "dx" in {
[IntrConvergent, IntrNoMem,
IntrTriviallyScalarizable]>;
+ def int_dx_wave_multi_prefix_bit_count
+ : DefaultAttrsIntrinsic<[llvm_i32_ty],
+ [llvm_i1_ty, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem]>;
+
def int_dx_wave_prefix_usum
: DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>],
[IntrConvergent, IntrNoMem,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index cf3d6404f6210..508680d51603c 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -1589,6 +1589,15 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> {
let attributes = [Attributes<DXIL1_5, []>];
}
+def WaveMultiPrefixBitCount : DXILOp<167, waveMultiPrefixBitCount> {
+ let Doc = "returns the prefix count for lanes selected by a mask";
+ let intrinsics = [IntrinSelect<int_dx_wave_multi_prefix_bit_count>];
+ let arguments = [Int1Ty, Int32Ty, Int32Ty, Int32Ty, Int32Ty];
+ let result = Int32Ty;
+ let stages = [Stages<DXIL1_5, [all_stages]>];
+ let attributes = [Attributes<DXIL1_5, []>];
+}
+
def AnnotateHandle : DXILOp<216, annotateHandle> {
let Doc = "annotate handle with resource properties";
let arguments = [HandleTy, ResPropsTy];
diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
index 1f37844dc5f19..7cefe50ea67cc 100644
--- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
+++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
@@ -113,6 +113,7 @@ static bool checkWaveOps(Intrinsic::ID IID) {
case Intrinsic::dx_wave_prefix_uproduct:
case Intrinsic::dx_wave_multi_prefix_bit_and:
case Intrinsic::dx_wave_multi_prefix_bit_xor:
+ case Intrinsic::dx_wave_multi_prefix_bit_count:
// Quad Op Variants
case Intrinsic::dx_quad_read_across_x:
case Intrinsic::dx_quad_read_across_y:
diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
index 833b5fee3d946..9c9d9dc24d933 100644
--- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
+++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
@@ -84,6 +84,13 @@ define i32 @wave_multi_prefix_bit_xor(i32 %x) {
ret i32 %ret
}
+define i32 @wave_multi_prefix_count_bits(i1 %x) {
+ ; CHECK: Function wave_multi_prefix_count_bits : [[WAVE_FLAG]]
+ %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+ i1 %x, i32 1, i32 2, i32 4, i32 8)
+ ret i32 %ret
+}
+
define noundef i1 @wave_all_equal(i1 %x) {
entry:
; CHECK: Function wave_all_equal : [[WAVE_FLAG]]
diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll
new file mode 100644
index 0000000000000..17ef364e41111
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -dxil-op-lower \
+; RUN: -mtriple=dxil-pc-shadermodel6.5-library < %s | FileCheck %s
+
+define i32 @test_dynamic(i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) {
+ ; CHECK-LABEL: define i32 @test_dynamic(
+ ; CHECK: %[[RET:.*]] = call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+ ; CHECK: ret i32 %[[RET]]
+ %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+ i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+ ret i32 %ret
+}
+
+define i32 @test_constant() {
+ ; CHECK-LABEL: define i32 @test_constant(
+ ; CHECK: %[[RET:.*]] = call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 true, i32 1, i32 2, i32 4, i32 8)
+ ; CHECK: ret i32 %[[RET]]
+ %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+ i1 true, i32 1, i32 2, i32 4, i32 8)
+ ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll
new file mode 100644
index 0000000000000..74b9c30732adc
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll
@@ -0,0 +1,9 @@
+; RUN: not opt -S -dxil-op-lower \
+; RUN: -mtriple=dxil-pc-shadermodel6.4-library < %s 2>&1 | FileCheck %s
+
+; CHECK: Cannot create WaveMultiPrefixBitCount operation: No valid stage for DXIL version 1.4
+define i32 @test_version(i1 %value) {
+ %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+ i1 %value, i32 1, i32 2, i32 4, i32 8)
+ ret i32 %ret
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/238132
More information about the llvm-branch-commits
mailing list