[clang] [llvm] [HLSL] Add WaveMultiPrefixBitAnd (PR #238083)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Nov 2 12:03:51 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-hlsl
@llvm/pr-subscribers-clang-codegen
Author: Joshua Batista (bob80905)
<details>
<summary>Changes</summary>
Add WaveMultiPrefixBitAnd for Shader Model 6.5.
Add Clang type checks and code generation for integer scalar, vector, and matrix values. Add DXIL lowering and SPIR-V partitioned subgroup scans. The SPIR-V intrinsic uses the subgroup name and requires SPV_NV_shader_subgroup_partitioned.
This is the first PR in a planned stack of five WaveMultiPrefix builtins.
An existing SPIR-V issue affects five-element vector function parameters. Memory-based long vectors work. This PR does not change that separate issue.
Fixes #<!-- -->99208.
---
Patch is 31.10 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/238083.diff
19 Files Affected:
- (modified) clang/include/clang/Basic/Builtins.td (+6)
- (modified) clang/include/clang/Basic/HLSLIntrinsics.td (+10)
- (modified) clang/lib/CodeGen/CGHLSLBuiltins.cpp (+10)
- (modified) clang/lib/CodeGen/CGHLSLRuntime.h (+3)
- (modified) clang/lib/Sema/SemaHLSL.cpp (+30)
- (added) clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl (+97)
- (added) clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl (+64)
- (modified) llvm/include/llvm/IR/IntrinsicsDirectX.td (+8)
- (modified) llvm/include/llvm/IR/IntrinsicsSPIRV.td (+7)
- (modified) llvm/lib/Target/DirectX/DXIL.td (+22)
- (modified) llvm/lib/Target/DirectX/DXILShaderFlags.cpp (+1)
- (modified) llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp (+2)
- (modified) llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp (+39)
- (modified) llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp (+1)
- (modified) llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td (+3-1)
- (modified) llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll (+7)
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd.ll (+48)
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixBitAnd_errors.ll (+14)
- (added) llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitAnd.ll (+85)
``````````diff
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index d04fd899eb63a6..5ae3edcc6601c3 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5789,6 +5789,12 @@ def HLSLWavePrefixSum : LangBuiltin<"HLSL_LANG"> {
let Prototype = "void(...)";
}
+def HLSLWaveMultiPrefixBitAnd : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_wave_multi_prefix_bit_and"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "void(...)";
+}
+
def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_wave_prefix_product"];
let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td
index fd59e3477b8c79..902a31d8278b37 100644
--- a/clang/include/clang/Basic/HLSLIntrinsics.td
+++ b/clang/include/clang/Basic/HLSLIntrinsics.td
@@ -1883,6 +1883,16 @@ def hlsl_wave_is_first_lane :
let Availability = SM6_0;
}
+def hlsl_wave_multi_prefix_bit_and :
+ HLSLTwoArgBuiltin<"WaveMultiPrefixBitAnd",
+ "__builtin_hlsl_wave_multi_prefix_bit_and"> {
+ let Args = [Varying, VectorType<UIntTy, 4>];
+ let VaryingTypes = AllIntTypes;
+ let VaryingLongVector = 1;
+ let IsConvergent = 1;
+ let Availability = SM6_5;
+}
+
// Returns the count of bits set to 1 on prior lanes.
def hlsl_wave_prefix_count_bits :
HLSLBuiltin<"WavePrefixCountBits",
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index fc422918499ed2..d4f5f47d9b24e9 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -1721,6 +1721,16 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
return EmitIntrinsicCall(IID, {OpExpr->getType()}, ArrayRef{OpExpr},
"hlsl.wave.prefix.sum");
}
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: {
+ Value *ValueOp = EmitScalarExpr(E->getArg(0));
+ Value *Mask = EmitScalarExpr(E->getArg(1));
+ SmallVector<Value *, 5> Args{ValueOp};
+ for (unsigned I = 0; I != 4; ++I)
+ Args.push_back(Builder.CreateExtractElement(Mask, I));
+ return EmitIntrinsicCall(
+ CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
+ {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.and");
+ }
case Builtin::BI__builtin_hlsl_wave_prefix_product: {
Value *OpExpr = EmitScalarExpr(E->getArg(0));
Intrinsic::ID IID = getWavePrefixProductIntrinsic(
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index 1957b946826d37..89b52f287d9ed7 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -149,6 +149,9 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and)
+ GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixBitAnd,
+ wave_multi_prefix_bit_and,
+ subgroup_multi_prefix_bit_and)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 746c45662030a1..6147a0446a838a 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -5377,6 +5377,36 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
TheCall->setType(ArgTyExpr);
break;
}
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: {
+ if (SemaRef.checkArgCount(TheCall, 2))
+ return true;
+
+ if (CheckAnyScalarOrVectorOrMatrix(&SemaRef, TheCall, 0) ||
+ CheckNotBoolScalarOrVector(&SemaRef, TheCall, 0))
+ return true;
+
+ QualType ValueTy = TheCall->getArg(0)->getType();
+ QualType ElementTy = getScalarComponentType(ValueTy);
+ if (!ElementTy->isIntegerType() || ElementTy->isBooleanType()) {
+ SemaRef.Diag(TheCall->getArg(0)->getBeginLoc(),
+ diag::err_builtin_invalid_arg_type)
+ << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << 0 << ValueTy;
+ return true;
+ }
+
+ QualType MaskTy = TheCall->getArg(1)->getType();
+ QualType UInt4Ty =
+ getASTContext().getExtVectorType(getASTContext().UnsignedIntTy, 4);
+ if (!getASTContext().hasSameUnqualifiedType(MaskTy, UInt4Ty)) {
+ SemaRef.Diag(TheCall->getArg(1)->getBeginLoc(),
+ diag::err_typecheck_convert_incompatible)
+ << MaskTy << UInt4Ty << 1 << 0 << 0;
+ return true;
+ }
+
+ TheCall->setType(ValueTy);
+ break;
+ }
case Builtin::BI__builtin_hlsl_quad_read_across_x:
case Builtin::BI__builtin_hlsl_quad_read_across_y:
case Builtin::BI__builtin_hlsl_quad_read_across_diagonal: {
diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl
new file mode 100644
index 00000000000000..d266d9752b8471
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitAnd.hlsl
@@ -0,0 +1,97 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | \
+// RUN: FileCheck %s -DINTRINSIC=dx.wave.multi.prefix.bit.and
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple spirv-pc-vulkan-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | \
+// RUN: FileCheck %s -DINTRINSIC=spv.subgroup.multi.prefix.bit.and
+
+// CHECK-LABEL: test_uint
+uint test_uint(uint value, uint4 mask) {
+ // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry()
+ // CHECK: %[[MASK:.*]] = load <4 x i32>
+ // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0
+ // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1
+ // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2
+ // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3
+ // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ]
+ // CHECK: ret i32 %[[RET]]
+ return WaveMultiPrefixBitAnd(value, mask);
+}
+
+// CHECK: declare i32 @llvm.[[INTRINSIC]].i32(i32, i32, i32, i32, i32) #[[ATTR:[0-9]+]]
+
+// CHECK-LABEL: test_int
+int test_int(int value) {
+ // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i32 %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int16
+int16_t test_int16(int16_t value) {
+ // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i16 %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint16
+uint16_t test_uint16(uint16_t value) {
+ // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i16 %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int64
+int64_t test_int64(int64_t value) {
+ // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i64 %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint64
+uint64_t test_uint64(uint64_t value) {
+ // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i64 %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint2
+uint2 test_uint2(uint2 value) {
+ // CHECK: %[[RET:.*]] = call <2 x i32> @llvm.[[INTRINSIC]].v2i32(<2 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <2 x i32> %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int3
+int3 test_int3(int3 value) {
+ // CHECK: %[[RET:.*]] = call <3 x i32> @llvm.[[INTRINSIC]].v3i32(<3 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <3 x i32> %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint64_t4
+uint64_t4 test_uint64_t4(uint64_t4 value) {
+ // CHECK: %[[RET:.*]] = call <4 x i64> @llvm.[[INTRINSIC]].v4i64(<4 x i64> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <4 x i64> %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint5
+vector<uint, 5> test_uint5(vector<uint, 5> value) {
+ // CHECK: %[[RET:.*]] = call <5 x i32> @llvm.[[INTRINSIC]].v5i32(<5 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <5 x i32> %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int2x3
+int2x3 test_int2x3(int2x3 value) {
+ // CHECK: %[[RET:.*]] = call <6 x i32> @llvm.[[INTRINSIC]].v6i32(<6 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <6 x i32> %[[RET]]
+ return WaveMultiPrefixBitAnd(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK: attributes #[[ATTR]] = { {{.*}}convergent {{.*}} }
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl
new file mode 100644
index 00000000000000..ae4142001c037e
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitAnd-errors.hlsl
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple spirv-pc-vulkan-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \
+// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s
+
+#ifdef TEST_AVAILABILITY
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main() {
+ (void)WaveMultiPrefixBitAnd(1u, uint4(1, 2, 4, 8));
+ // availability-error at -1 {{'WaveMultiPrefixBitAnd' is only available on Shader Model 6.5 or newer}}
+ // availability-note at hlsl/hlsl_alias_intrinsics_gen.inc:* {{'WaveMultiPrefixBitAnd' has been marked as being introduced in Shader Model 6.5 here}}
+}
+#else
+void test_arg_count(uint value, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_bit_and(value);
+ // expected-error at -1 {{too few arguments to function call, expected 2, have 1}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(value, mask, mask);
+ // expected-error at -1 {{too many arguments to function call, expected 2, have 3}}
+}
+
+struct S { uint value; };
+
+void test_value_types(float f, float2 fv, bool b, bool2 bv, S s,
+ float2x2 fm, bool2x2 bm, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_bit_and(f, mask);
+ // expected-error at -1 {{1st argument must be a scalar or vector of integer}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(fv, mask);
+ // expected-error at -1 {{1st argument must be a scalar or vector of integer}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(b, mask);
+ // expected-error at -1 {{invalid operand of type 'bool'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(bv, mask);
+ // expected-error at -1 {{invalid operand of type 'bool2'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(s, mask);
+ // expected-error at -1 {{invalid operand of type 'S'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(fm, mask);
+ // expected-error at -1 {{1st argument must be a matrix integer type}}
+ // expected-error at -2 {{cannot pass object of non-trivial type 'float2x2'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(bm, mask);
+ // expected-error at -1 {{1st argument must be a matrix integer type}}
+ // expected-error at -2 {{cannot pass object of non-trivial type 'bool2x2'}}
+}
+
+void test_mask_types(uint value, uint scalar, uint3 short_mask,
+ int4 signed_mask, float4 float_mask,
+ uint64_t4 wide_mask, S struct_mask) {
+ __builtin_hlsl_wave_multi_prefix_bit_and(value, scalar);
+ // expected-error at -1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'vector<unsigned int, 4>'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(value, short_mask);
+ // expected-error at -1 {{passing 'uint3' (aka 'vector<uint, 3>') to parameter of incompatible type 'vector<unsigned int, 4>'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(value, signed_mask);
+ // expected-error at -1 {{passing 'int4' (aka 'vector<int, 4>') to parameter of incompatible type 'vector<unsigned int, 4>'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(value, float_mask);
+ // expected-error at -1 {{passing 'float4' (aka 'vector<float, 4>') to parameter of incompatible type 'vector<unsigned int, 4>'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(value, wide_mask);
+ // expected-error at -1 {{passing 'uint64_t4'}}
+ __builtin_hlsl_wave_multi_prefix_bit_and(value, struct_mask);
+ // expected-error at -1 {{passing 'S' to parameter of incompatible type 'vector<unsigned int, 4>'}}
+}
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index b1cb80cd96f143..4f23541e7eeed0 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -473,6 +473,14 @@ let TargetPrefix = "dx" in {
[IntrConvergent, IntrNoMem,
IntrTriviallyScalarizable]>;
+ // Scalar mask words preserve the lane mask during vector scalarization.
+ def int_dx_wave_multi_prefix_bit_and
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem,
+ IntrTriviallyScalarizable]>;
+
def int_dx_wave_prefix_usum
: DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>],
[IntrConvergent, IntrNoMem,
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index f9319918694d7d..2b1127445f7990 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -383,6 +383,13 @@ let TargetPrefix = "spv" in {
[IntrConvergent, IntrNoMem,
IntrTriviallyScalarizable]>;
+ def int_spv_subgroup_multi_prefix_bit_and
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem,
+ IntrTriviallyScalarizable]>;
+
def int_spv_wave_prefix_product
: DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>],
[IntrConvergent, IntrNoMem,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index a3a373d79f73cb..c2165c9664a3d6 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -324,6 +324,8 @@ defvar WaveBitOpKind_And = 0;
defvar WaveBitOpKind_Or = 1;
defvar WaveBitOpKind_Xor = 2;
+defvar WaveMultiPrefixOpKind_And = 1;
+
defvar SignedOpKind_Signed = 0;
defvar SignedOpKind_Unsigned = 1;
@@ -1559,6 +1561,26 @@ def Dot4AddU8Packed : DXILOp<164, dot4AddPacked> {
let attributes = [Attributes<DXIL1_4, [DXILReadNone]>];
}
+def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> {
+ let Doc = "returns the prefix result for lanes selected by a mask";
+ let intrinsics = [
+ IntrinSelect<int_dx_wave_multi_prefix_bit_and,
+ [
+ IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>,
+ IntrinArgIndex<3>, IntrinArgIndex<4>,
+ IntrinArgI8<WaveMultiPrefixOpKind_And>,
+ IntrinArgI8<SignedOpKind_Signed>
+ ]>
+ ];
+ let arguments = [
+ OverloadTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, Int8Ty, Int8Ty
+ ];
+ let result = OverloadTy;
+ let overloads = [Overloads<DXIL1_5, [Int16Ty, Int32Ty, Int64Ty]>];
+ let stages = [Stages<DXIL1_5, [all_stages]>];
+ let attributes = [Attributes<DXIL1_5, []>];
+}
+
def AnnotateHandle : DXILOp<216, annotateHandle> {
let Doc = "annotate handle with resource properties";
let arguments = [HandleTy, ResPropsTy];
diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
index 9c709cf5a1ca1c..1c27d16e05979b 100644
--- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
+++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
@@ -111,6 +111,7 @@ static bool checkWaveOps(Intrinsic::ID IID) {
case Intrinsic::dx_wave_prefix_usum:
case Intrinsic::dx_wave_prefix_product:
case Intrinsic::dx_wave_prefix_uproduct:
+ case Intrinsic::dx_wave_multi_prefix_bit_and:
// Quad Op Variants
case Intrinsic::dx_quad_read_across_x:
case Intrinsic::dx_quad_read_across_y:
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index 6c65fb7b9e0fc7..6fa1d275334423 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -20,6 +20,8 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg(
switch (ID) {
case Intrinsic::dx_wave_readlane:
return ScalarOpdIdx == 1;
+ case Intrinsic::dx_wave_multi_prefix_bit_and:
+ return ScalarOpdIdx >= 1 && ScalarOpdIdx <= 4;
default:
return false;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 71040b3b85faf1..ea7723cea88fef 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -322,6 +322,9 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
+ bool selectWaveMultiPrefixBitAnd(Register ResVReg, SPIRVTypeInst ResType,
+ MachineInstr &I) const;
+
bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
unsigned Direction) const;
@@ -4011,6 +4014,40 @@ bool SPIRVInstructionSelector::selectWaveExclusiveScan(
return true;
}
+bool SPIRVInstructionSelector::selectWaveMultiPrefixBitAnd(
+ Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
+ if (!STI.canUseExtension(
+ SPIRV::Extension::SPV_NV_shader_subgroup_partitioned))
+ return diagnoseUnsupported(
+ I, "WaveMultiPrefixBitAnd requires the SPIR-V extension "
+ "SPV_NV_shader_subgroup_partitioned");
+
+ assert(I.getNumOperands() == 7);
+ SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
+ SPIRVTypeInst MaskTy = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII);
+ Register Mask = MRI->createVirtualRegister(GR.getRegClass(MaskTy));
+ GR.assignSPIRVTypeToVReg(MaskTy, Mask, MRI->getMF());
+ auto MaskMIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
+ TII.get(SPIRV::OpCompositeConstruct))
+ .addDef(Mask)
+ .addUse(GR.getSPIRVTypeID(MaskTy));
+ for (unsigned Op = 3; Op != 7; ++Op)
+ MaskMIB.addUse(I.getOperand(Op).getReg());
+ MaskMIB.constrainAllUses(TII, TRI, RBI);
+
+ BuildMI(*I.getParent(), I, I.getDebugLoc(),
+ TII.get(SPIRV::OpGroupNonUniformBitwiseAnd))
+ .addDef(ResVReg)
+ .addUse(GR.getSPIRVTypeID(ResType))
+ .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
+ !STI.isShader()))
+ .addImm(SPIRV::GroupOperation::PartitionedExclusiveScanNV)
+ .addUse(I.getOperand(2).getReg())
+ .addUse(Mask)
+ .constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
bool SPIRVInstructionSelector::selectQuadSwap(Register ResVReg,
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/238083
More information about the cfe-commits
mailing list