[llvm-branch-commits] [clang] [llvm] [HLSL] Add WaveMultiPrefixCountBits (PR #238132)

via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Mon Nov 2 15:00:11 PST 2026


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-directx

Author: Joshua Batista (bob80905)

<details>
<summary>Changes</summary>

This is the third PR in the WaveMultiPrefix stack. It builds on #<!-- -->238097.

This PR adds Clang and DXIL support for `WaveMultiPrefixCountBits(bool value, uint4 mask)` in Shader Model 6.5 or newer. It returns a `uint` and lowers to DXIL opcode 167, `WaveMultiPrefixBitCount`.

This PR does not add SPIR-V support. SPIR-V calls report that the builtin is not supported on the target.

Fixes #<!-- -->99211.

---
Full diff: https://github.com/llvm/llvm-project/pull/238132.diff


12 Files Affected:

- (modified) clang/include/clang/Basic/Builtins.td (+6) 
- (modified) clang/include/clang/Basic/HLSLIntrinsics.td (+9) 
- (modified) clang/lib/CodeGen/CGHLSLBuiltins.cpp (+4) 
- (modified) clang/lib/Sema/SemaHLSL.cpp (+32-10) 
- (added) clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl (+45) 
- (added) clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl (+63) 
- (modified) llvm/include/llvm/IR/IntrinsicsDirectX.td (+6) 
- (modified) llvm/lib/Target/DirectX/DXIL.td (+9) 
- (modified) llvm/lib/Target/DirectX/DXILShaderFlags.cpp (+1) 
- (modified) llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll (+7) 
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll (+20) 
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll (+9) 


``````````diff
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index ab4b7f4b1187a..e2679df195475 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5801,6 +5801,12 @@ def HLSLWaveMultiPrefixBitXor : LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void(...)";
 }
 
+def HLSLWaveMultiPrefixCountBits : LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_wave_multi_prefix_count_bits"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "unsigned int(bool, _ExtVector<4, unsigned int>)";
+}
+
 def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_wave_prefix_product"];
   let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td
index 300d46a7fd324..89728b4ba885c 100644
--- a/clang/include/clang/Basic/HLSLIntrinsics.td
+++ b/clang/include/clang/Basic/HLSLIntrinsics.td
@@ -1903,6 +1903,15 @@ def hlsl_wave_multi_prefix_bit_xor :
   let Availability = SM6_5;
 }
 
+def hlsl_wave_multi_prefix_count_bits :
+    HLSLBuiltin<"WaveMultiPrefixCountBits",
+                "__builtin_hlsl_wave_multi_prefix_count_bits"> {
+  let Args = [BoolTy, VectorType<UIntTy, 4>];
+  let ReturnType = UIntTy;
+  let IsConvergent = 1;
+  let Availability = SM6_5;
+}
+
 // Returns the count of bits set to 1 on prior lanes.
 def hlsl_wave_prefix_count_bits :
     HLSLBuiltin<"WavePrefixCountBits",
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index 3afc8a0668f27..229a1cc4f830a 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -1721,6 +1721,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     return EmitIntrinsicCall(IID, {OpExpr->getType()}, ArrayRef{OpExpr},
                              "hlsl.wave.prefix.sum");
   }
+  case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits:
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: {
     Value *ValueOp = EmitScalarExpr(E->getArg(0));
@@ -1728,6 +1729,9 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     SmallVector<Value *, 5> Args{ValueOp};
     for (unsigned I = 0; I != 4; ++I)
       Args.push_back(Builder.CreateExtractElement(Mask, I));
+    if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits)
+      return EmitIntrinsicCall(Intrinsic::dx_wave_multi_prefix_bit_count, Args,
+                               "hlsl.wave.multi.prefix.bit.count");
     if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and)
       return EmitIntrinsicCall(
           CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index d0da34ef79f89..e0a92ff1d472c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -4012,6 +4012,18 @@ static bool CheckWavePrefix(Sema *S, CallExpr *TheCall) {
   return false;
 }
 
+static bool CheckWaveMultiPrefixMask(Sema *S, CallExpr *TheCall) {
+  QualType MaskTy = TheCall->getArg(1)->getType();
+  QualType UInt4Ty = S->Context.getExtVectorType(S->Context.UnsignedIntTy, 4);
+  if (S->Context.hasSameUnqualifiedType(MaskTy, UInt4Ty))
+    return false;
+
+  S->Diag(TheCall->getArg(1)->getBeginLoc(),
+          diag::err_typecheck_convert_incompatible)
+      << MaskTy << UInt4Ty << 1 << 0 << 0;
+  return true;
+}
+
 static bool CheckBoolSelect(Sema *S, CallExpr *TheCall) {
   assert(TheCall->getNumArgs() == 3);
   Expr *Arg1 = TheCall->getArg(1);
@@ -5302,8 +5314,11 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
       return true;
     break;
   }
-  case Builtin::BI__builtin_hlsl_wave_prefix_count_bits: {
-    if (SemaRef.checkArgCount(TheCall, 1))
+  case Builtin::BI__builtin_hlsl_wave_prefix_count_bits:
+  case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits: {
+    bool IsMultiPrefix =
+        BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits;
+    if (SemaRef.checkArgCount(TheCall, IsMultiPrefix ? 2 : 1))
       return true;
 
     QualType ArgType = TheCall->getArg(0)->getType();
@@ -5322,6 +5337,20 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
       return true;
     }
 
+    if (IsMultiPrefix) {
+      if (CheckWaveMultiPrefixMask(&SemaRef, TheCall))
+        return true;
+
+      // Alias validation uses calls with an invalid source location.
+      if (TheCall->getBeginLoc().isValid() &&
+          !getASTContext().getTargetInfo().getTriple().isDXIL()) {
+        SemaRef.Diag(TheCall->getBeginLoc(),
+                     diag::err_builtin_target_unsupported)
+            << TheCall->getSourceRange();
+        return true;
+      }
+    }
+
     break;
   }
   case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
@@ -5395,15 +5424,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
       return true;
     }
 
-    QualType MaskTy = TheCall->getArg(1)->getType();
-    QualType UInt4Ty =
-        getASTContext().getExtVectorType(getASTContext().UnsignedIntTy, 4);
-    if (!getASTContext().hasSameUnqualifiedType(MaskTy, UInt4Ty)) {
-      SemaRef.Diag(TheCall->getArg(1)->getBeginLoc(),
-                   diag::err_typecheck_convert_incompatible)
-          << MaskTy << UInt4Ty << 1 << 0 << 0;
+    if (CheckWaveMultiPrefixMask(&SemaRef, TheCall))
       return true;
-    }
 
     TheCall->setType(ValueTy);
     break;
diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl
new file mode 100644
index 0000000000000..4b7c8445d5c7e
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixCountBits.hlsl
@@ -0,0 +1,45 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN:   -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN:   -disable-llvm-passes -o - | FileCheck %s
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN:   -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN:   -disable-llvm-passes -o - | \
+// RUN:   opt -S -strip-convergence-intrinsics -dxil-op-lower | \
+// RUN:   FileCheck %s --check-prefix=DXIL
+
+// CHECK-LABEL: test_bool
+// DXIL-LABEL: test_bool
+uint test_bool(bool value, uint4 mask) {
+  // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry()
+  // CHECK: %[[VALUE:.*]] = icmp ne i32 %{{.*}}, 0
+  // CHECK: %[[MASK:.*]] = load <4 x i32>
+  // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0
+  // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1
+  // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2
+  // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3
+  // CHECK: %[[RET:.*]] = call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 %[[VALUE]], i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ]
+  // CHECK: ret i32 %[[RET]]
+  // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+  return WaveMultiPrefixCountBits(value, mask);
+}
+
+// CHECK: declare i32 @llvm.dx.wave.multi.prefix.bit.count(i1, i32, i32, i32, i32) #[[ATTR:[0-9]+]]
+
+// CHECK-LABEL: test_true
+// DXIL-LABEL: test_true
+uint test_true() {
+  // CHECK: call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 true, i32 1, i32 2, i32 4, i32 8)
+  // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 true, i32 1, i32 2, i32 4, i32 8)
+  return WaveMultiPrefixCountBits(true, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_false
+// DXIL-LABEL: test_false
+uint test_false() {
+  // CHECK: call i32 @llvm.dx.wave.multi.prefix.bit.count(i1 false, i32 16, i32 32, i32 64, i32 128)
+  // DXIL: call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 false, i32 16, i32 32, i32 64, i32 128)
+  return __builtin_hlsl_wave_multi_prefix_count_bits(
+      false, uint4(16, 32, 64, 128));
+}
+
+// CHECK: attributes #[[ATTR]] = { convergent {{.*}}memory(none) }
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl
new file mode 100644
index 0000000000000..4b0944d349395
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixCountBits-errors.hlsl
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN:   -triple dxil-pc-shadermodel6.5-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \
+// RUN:   -triple dxil-pc-shadermodel6.4-library -verify=availability %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_UNSUPPORTED \
+// RUN:   -triple spirv-pc-vulkan-library -verify=spirv %s
+
+#ifdef TEST_AVAILABILITY
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main() {
+  (void)WaveMultiPrefixCountBits(true, uint4(1, 2, 4, 8));
+  // availability-error at -1 {{only available on Shader Model 6.5 or newer}}
+  // availability-note@*:* {{introduced in Shader Model 6.5}}
+}
+#elif defined(TEST_UNSUPPORTED)
+uint test_public(bool value, uint4 mask) {
+  return WaveMultiPrefixCountBits(value, mask);
+  // spirv-error at -1 {{builtin is not supported on this target}}
+}
+
+uint test_builtin(bool value, uint4 mask) {
+  return __builtin_hlsl_wave_multi_prefix_count_bits(value, mask);
+  // spirv-error at -1 {{builtin is not supported on this target}}
+}
+#else
+void test_arg_count(bool value, uint4 mask) {
+  __builtin_hlsl_wave_multi_prefix_count_bits(value);
+  // expected-error at -1 {{too few arguments to function call, expected 2}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(value, mask, mask);
+  // expected-error at -1 {{too many arguments to function call, expected 2}}
+}
+
+struct S { uint value; };
+
+void test_value_types(float f, int i, bool2 bv, S s, uint4 mask) {
+  __builtin_hlsl_wave_multi_prefix_count_bits(f, mask);
+  // expected-error at -1 {{invalid operand of type 'float'}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(i, mask);
+  // expected-error at -1 {{invalid operand of type 'int'}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(bv, mask);
+  // expected-error at -1 {{invalid operand of type 'bool2'}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(s, mask);
+  // expected-error at -1 {{invalid operand of type 'S'}}
+}
+
+void test_mask_types(bool value, uint scalar, uint3 short_mask,
+                     int4 signed_mask, float4 float_mask,
+                     uint64_t4 wide_mask, S struct_mask) {
+  __builtin_hlsl_wave_multi_prefix_count_bits(value, scalar);
+  // expected-error at -1 {{passing 'uint' (aka 'unsigned int')}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(value, short_mask);
+  // expected-error at -1 {{passing 'uint3' (aka 'vector<uint, 3>')}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(value, signed_mask);
+  // expected-error at -1 {{passing 'int4' (aka 'vector<int, 4>')}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(value, float_mask);
+  // expected-error at -1 {{passing 'float4' (aka 'vector<float, 4>')}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(value, wide_mask);
+  // expected-error at -1 {{passing 'uint64_t4'}}
+  __builtin_hlsl_wave_multi_prefix_count_bits(value, struct_mask);
+  // expected-error at -1 {{passing 'S' to parameter of incompatible type}}
+}
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index c292a0a46b2ae..6343cd2332bd2 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -488,6 +488,12 @@ let TargetPrefix = "dx" in {
                               [IntrConvergent, IntrNoMem,
                                IntrTriviallyScalarizable]>;
 
+  def int_dx_wave_multi_prefix_bit_count
+      : DefaultAttrsIntrinsic<[llvm_i32_ty],
+                              [llvm_i1_ty, llvm_i32_ty, llvm_i32_ty,
+                               llvm_i32_ty, llvm_i32_ty],
+                              [IntrConvergent, IntrNoMem]>;
+
   def int_dx_wave_prefix_usum
       : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>],
                               [IntrConvergent, IntrNoMem,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index cf3d6404f6210..508680d51603c 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -1589,6 +1589,15 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> {
   let attributes = [Attributes<DXIL1_5, []>];
 }
 
+def WaveMultiPrefixBitCount : DXILOp<167, waveMultiPrefixBitCount> {
+  let Doc = "returns the prefix count for lanes selected by a mask";
+  let intrinsics = [IntrinSelect<int_dx_wave_multi_prefix_bit_count>];
+  let arguments = [Int1Ty, Int32Ty, Int32Ty, Int32Ty, Int32Ty];
+  let result = Int32Ty;
+  let stages = [Stages<DXIL1_5, [all_stages]>];
+  let attributes = [Attributes<DXIL1_5, []>];
+}
+
 def AnnotateHandle : DXILOp<216, annotateHandle> {
   let Doc = "annotate handle with resource properties";
   let arguments = [HandleTy, ResPropsTy];
diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
index 1f37844dc5f19..7cefe50ea67cc 100644
--- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
+++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
@@ -113,6 +113,7 @@ static bool checkWaveOps(Intrinsic::ID IID) {
   case Intrinsic::dx_wave_prefix_uproduct:
   case Intrinsic::dx_wave_multi_prefix_bit_and:
   case Intrinsic::dx_wave_multi_prefix_bit_xor:
+  case Intrinsic::dx_wave_multi_prefix_bit_count:
     // Quad Op Variants
   case Intrinsic::dx_quad_read_across_x:
   case Intrinsic::dx_quad_read_across_y:
diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
index 833b5fee3d946..9c9d9dc24d933 100644
--- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
+++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
@@ -84,6 +84,13 @@ define i32 @wave_multi_prefix_bit_xor(i32 %x) {
   ret i32 %ret
 }
 
+define i32 @wave_multi_prefix_count_bits(i1 %x) {
+  ; CHECK: Function wave_multi_prefix_count_bits : [[WAVE_FLAG]]
+  %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+      i1 %x, i32 1, i32 2, i32 4, i32 8)
+  ret i32 %ret
+}
+
 define noundef i1 @wave_all_equal(i1 %x) {
 entry:
   ; CHECK: Function wave_all_equal : [[WAVE_FLAG]]
diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll
new file mode 100644
index 0000000000000..17ef364e41111
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -dxil-op-lower \
+; RUN:   -mtriple=dxil-pc-shadermodel6.5-library < %s | FileCheck %s
+
+define i32 @test_dynamic(i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) {
+  ; CHECK-LABEL: define i32 @test_dynamic(
+  ; CHECK: %[[RET:.*]] = call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+  ; CHECK: ret i32 %[[RET]]
+  %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+      i1 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+  ret i32 %ret
+}
+
+define i32 @test_constant() {
+  ; CHECK-LABEL: define i32 @test_constant(
+  ; CHECK: %[[RET:.*]] = call i32 @dx.op.waveMultiPrefixBitCount(i32 167, i1 true, i32 1, i32 2, i32 4, i32 8)
+  ; CHECK: ret i32 %[[RET]]
+  %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+      i1 true, i32 1, i32 2, i32 4, i32 8)
+  ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll
new file mode 100644
index 0000000000000..74b9c30732adc
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixCountBits_errors.ll
@@ -0,0 +1,9 @@
+; RUN: not opt -S -dxil-op-lower \
+; RUN:   -mtriple=dxil-pc-shadermodel6.4-library < %s 2>&1 | FileCheck %s
+
+; CHECK: Cannot create WaveMultiPrefixBitCount operation: No valid stage for DXIL version 1.4
+define i32 @test_version(i1 %value) {
+  %ret = call i32 @llvm.dx.wave.multi.prefix.bit.count(
+      i1 %value, i32 1, i32 2, i32 4, i32 8)
+  ret i32 %ret
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/238132


More information about the llvm-branch-commits mailing list