[llvm-branch-commits] [clang] [llvm] [HLSL] Add WaveMultiPrefixSum (PR #238175)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 2 17:03:16 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang-codegen
Author: Joshua Batista (bob80905)
<details>
<summary>Changes</summary>
Adds WaveMultiPrefixSum for Shader Model 6.5 and newer.
Supports numeric scalars, vectors, long vectors, and matrices in Clang. Adds DXIL lowering with the Sum operation value 0 and the correct signedness. Adds SPIR-V subgroup lowering with either the EXT or NV partition extension. Prefers EXT when both are enabled.
The existing SPIR-V limit on long-vector function parameters remains. The backend test uses memory for the long-vector case.
The build and all 33 focused tests pass. SPIR-V binaries pass spirv-val with NV, EXT, and both extensions.
Stacked on #<!-- -->238164.
Fixes #<!-- -->99213
---
Patch is 32.53 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/238175.diff
17 Files Affected:
- (modified) clang/include/clang/Basic/Builtins.td (+6)
- (modified) clang/include/clang/Basic/HLSLIntrinsics.td (+10)
- (modified) clang/lib/CodeGen/CGHLSLBuiltins.cpp (+9)
- (modified) clang/lib/CodeGen/CGHLSLRuntime.h (+6)
- (modified) clang/lib/Sema/SemaHLSL.cpp (+7-5)
- (added) clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl (+113)
- (added) clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl (+56)
- (modified) llvm/include/llvm/IR/IntrinsicsDirectX.td (+14)
- (modified) llvm/include/llvm/IR/IntrinsicsSPIRV.td (+7)
- (modified) llvm/lib/Target/DirectX/DXIL.td (+15)
- (modified) llvm/lib/Target/DirectX/DXILShaderFlags.cpp (+2)
- (modified) llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp (+2)
- (modified) llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp (+8)
- (modified) llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll (+14)
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixSum.ll (+81)
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixSum_errors.ll (+14)
- (added) llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixSum.ll (+147)
``````````diff
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 0d0bc61907fbb..ad5f1e10f34b1 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5813,6 +5813,12 @@ def HLSLWaveMultiPrefixProduct : LangBuiltin<"HLSL_LANG"> {
let Prototype = "void(...)";
}
+def HLSLWaveMultiPrefixSum : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_wave_multi_prefix_sum"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "void(...)";
+}
+
def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_wave_prefix_product"];
let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td
index 7009127c444b6..e87cabb5c865b 100644
--- a/clang/include/clang/Basic/HLSLIntrinsics.td
+++ b/clang/include/clang/Basic/HLSLIntrinsics.td
@@ -1922,6 +1922,16 @@ def hlsl_wave_multi_prefix_product :
let Availability = SM6_5;
}
+def hlsl_wave_multi_prefix_sum :
+ HLSLTwoArgBuiltin<"WaveMultiPrefixSum",
+ "__builtin_hlsl_wave_multi_prefix_sum"> {
+ let Args = [Varying, VectorType<UIntTy, 4>];
+ let VaryingTypes = AllNumericTypes;
+ let VaryingLongVector = 1;
+ let IsConvergent = 1;
+ let Availability = SM6_5;
+}
+
// Returns the count of bits set to 1 on prior lanes.
def hlsl_wave_prefix_count_bits :
HLSLBuiltin<"WavePrefixCountBits",
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index d301a16f5b2e1..38893d841d922 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -1723,6 +1723,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
}
case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits:
case Builtin::BI__builtin_hlsl_wave_multi_prefix_product:
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_sum:
case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: {
Value *ValueOp = EmitScalarExpr(E->getArg(0));
@@ -1741,6 +1742,14 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
return EmitIntrinsicCall(IID, {ValueOp->getType()}, Args,
"hlsl.wave.multi.prefix.product");
}
+ if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_sum) {
+ Intrinsic::ID IID =
+ E->getArg(0)->getType()->hasUnsignedIntegerRepresentation()
+ ? CGM.getHLSLRuntime().getWaveMultiPrefixUSumIntrinsic()
+ : CGM.getHLSLRuntime().getWaveMultiPrefixSumIntrinsic();
+ return EmitIntrinsicCall(IID, {ValueOp->getType()}, Args,
+ "hlsl.wave.multi.prefix.sum");
+ }
if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and)
return EmitIntrinsicCall(
CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index 50a7729be6875..35237123713c9 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -161,6 +161,12 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixUProduct,
wave_multi_prefix_uproduct,
subgroup_multi_prefix_product)
+ GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixSum,
+ wave_multi_prefix_sum,
+ subgroup_multi_prefix_sum)
+ GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixUSum,
+ wave_multi_prefix_usum,
+ subgroup_multi_prefix_sum)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index d6a280e1fb6cb..7b8d911d8309d 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -5408,7 +5408,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
}
case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor:
- case Builtin::BI__builtin_hlsl_wave_multi_prefix_product: {
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_product:
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_sum: {
if (SemaRef.checkArgCount(TheCall, 2))
return true;
@@ -5418,14 +5419,15 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
QualType ValueTy = TheCall->getArg(0)->getType();
QualType ElementTy = getScalarComponentType(ValueTy);
- bool IsProduct =
- BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_product;
+ bool IsArithmetic =
+ BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_product ||
+ BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_sum;
if ((!ElementTy->isIntegerType() &&
- !(IsProduct && ElementTy->hasFloatingRepresentation())) ||
+ !(IsArithmetic && ElementTy->hasFloatingRepresentation())) ||
ElementTy->isBooleanType()) {
SemaRef.Diag(TheCall->getArg(0)->getBeginLoc(),
diag::err_builtin_invalid_arg_type)
- << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << IsProduct
+ << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << IsArithmetic
<< ValueTy;
return true;
}
diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl
new file mode 100644
index 0000000000000..943a90884521d
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl
@@ -0,0 +1,113 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DINTRINSIC=dx.wave.multi.prefix.sum \
+// RUN: -DUINTRINSIC=dx.wave.multi.prefix.usum
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple spirv-pc-vulkan-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | FileCheck %s \
+// RUN: -DINTRINSIC=spv.subgroup.multi.prefix.sum \
+// RUN: -DUINTRINSIC=spv.subgroup.multi.prefix.sum
+
+// CHECK-LABEL: test_float
+float test_float(float value, uint4 mask) {
+ // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry()
+ // CHECK: %[[MASK:.*]] = load <4 x i32>
+ // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0
+ // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1
+ // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2
+ // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3
+ // CHECK: %[[RET:.*]] = call {{.*}}float @llvm.[[INTRINSIC]].f32(float %{{.*}}, i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ]
+ // CHECK: ret float %[[RET]]
+ return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK: declare float @llvm.[[INTRINSIC]].f32(float, i32, i32, i32, i32) #[[ATTR:[0-9]+]]
+
+// CHECK-LABEL: test_half
+float16_t test_half(float16_t value) {
+ // CHECK: call {{.*}}half @llvm.[[INTRINSIC]].f16(half %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_double
+double test_double(double value) {
+ // CHECK: call {{.*}}double @llvm.[[INTRINSIC]].f64(double %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int
+int test_int(int value) {
+ // CHECK: call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return __builtin_hlsl_wave_multi_prefix_sum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int16
+int16_t test_int16(int16_t value) {
+ // CHECK: call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int64
+int64_t test_int64(int64_t value) {
+ // CHECK: call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint
+uint test_uint(uint value) {
+ // CHECK: call i32 @llvm.[[UINTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint16
+uint16_t test_uint16(uint16_t value) {
+ // CHECK: call i16 @llvm.[[UINTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint64
+uint64_t test_uint64(uint64_t value) {
+ // CHECK: call i64 @llvm.[[UINTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint2
+uint2 test_uint2(uint2 value, uint4 mask) {
+ // CHECK: call <2 x i32> @llvm.[[UINTRINSIC]].v2i32(<2 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+ return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_float4
+float4 test_float4(float4 value, uint4 mask) {
+ // CHECK: call {{.*}}<4 x float> @llvm.[[INTRINSIC]].v4f32(<4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+ return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_uint_matrix
+uint2x2 test_uint_matrix(uint2x2 value, uint4 mask) {
+ // CHECK: call <4 x i32> @llvm.[[UINTRINSIC]].v4i32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+ return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_float_matrix
+float2x2 test_float_matrix(float2x2 value, uint4 mask) {
+ // CHECK: call {{.*}}<4 x float> @llvm.[[INTRINSIC]].v4f32(<4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+ return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_uint5
+vector<uint, 5> test_uint5(vector<uint, 5> value) {
+ // CHECK: call <5 x i32> @llvm.[[UINTRINSIC]].v5i32(<5 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_float5
+vector<float, 5> test_float5(vector<float, 5> value) {
+ // CHECK: call {{.*}}<5 x float> @llvm.[[INTRINSIC]].v5f32(<5 x float> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK: attributes #[[ATTR]] = { convergent {{.*}}memory(none) }
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl
new file mode 100644
index 0000000000000..f6ffeee2e1eaf
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl
@@ -0,0 +1,56 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple spirv-pc-vulkan-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \
+// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s
+
+#ifdef TEST_AVAILABILITY
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main() {
+ (void)WaveMultiPrefixSum(1.0f, uint4(1, 2, 4, 8));
+ // availability-error at -1 {{only available on Shader Model 6.5 or newer}}
+ // availability-note@*:* {{introduced in Shader Model 6.5}}
+}
+#else
+void test_arg_count(float value, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_sum(value);
+ // expected-error at -1 {{too few arguments to function call, expected 2}}
+ __builtin_hlsl_wave_multi_prefix_sum(value, mask, mask);
+ // expected-error at -1 {{too many arguments to function call, expected 2}}
+}
+
+struct S { float value; };
+
+void test_value_types(bool b, bool2 bv, bool2x2 bm, S s, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_sum(b, mask);
+ // expected-error at -1 {{invalid operand of type 'bool'}}
+ __builtin_hlsl_wave_multi_prefix_sum(bv, mask);
+ // expected-error at -1 {{invalid operand of type 'bool2'}}
+ __builtin_hlsl_wave_multi_prefix_sum(bm, mask);
+ // expected-error at -1 {{matrix, integer or floating-point}}
+ // expected-error at -2 {{cannot pass object of non-trivial type 'bool2x2'}}
+ __builtin_hlsl_wave_multi_prefix_sum(s, mask);
+ // expected-error at -1 {{invalid operand of type 'S'}}
+}
+
+void test_mask_types(float value, uint scalar, uint3 short_mask,
+ int4 signed_mask, float4 float_mask,
+ uint64_t4 wide_mask, S struct_mask) {
+ __builtin_hlsl_wave_multi_prefix_sum(value, scalar);
+ // expected-error at -1 {{passing 'uint' (aka 'unsigned int')}}
+ __builtin_hlsl_wave_multi_prefix_sum(value, short_mask);
+ // expected-error at -1 {{passing 'uint3' (aka 'vector<uint, 3>')}}
+ __builtin_hlsl_wave_multi_prefix_sum(value, signed_mask);
+ // expected-error at -1 {{passing 'int4' (aka 'vector<int, 4>')}}
+ __builtin_hlsl_wave_multi_prefix_sum(value, float_mask);
+ // expected-error at -1 {{passing 'float4' (aka 'vector<float, 4>')}}
+ __builtin_hlsl_wave_multi_prefix_sum(value, wide_mask);
+ // expected-error at -1 {{passing 'uint64_t4'}}
+ __builtin_hlsl_wave_multi_prefix_sum(value, struct_mask);
+ // expected-error at -1 {{passing 'S' to parameter of incompatible type}}
+}
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 1aaabaec97309..4a7b49831c9d4 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -508,6 +508,20 @@ let TargetPrefix = "dx" in {
[IntrConvergent, IntrNoMem,
IntrTriviallyScalarizable]>;
+ def int_dx_wave_multi_prefix_sum
+ : DefaultAttrsIntrinsic<[llvm_any_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem,
+ IntrTriviallyScalarizable]>;
+
+ def int_dx_wave_multi_prefix_usum
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem,
+ IntrTriviallyScalarizable]>;
+
def int_dx_wave_prefix_usum
: DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>],
[IntrConvergent, IntrNoMem,
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 926320653ab52..28e4d580bbcdf 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -404,6 +404,13 @@ let TargetPrefix = "spv" in {
[IntrConvergent, IntrNoMem,
IntrTriviallyScalarizable]>;
+ def int_spv_subgroup_multi_prefix_sum
+ : DefaultAttrsIntrinsic<[llvm_any_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem,
+ IntrTriviallyScalarizable]>;
+
def int_spv_wave_prefix_product
: DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>],
[IntrConvergent, IntrNoMem,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 361ddac0a1b3e..accd5be5801f1 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -324,6 +324,7 @@ defvar WaveBitOpKind_And = 0;
defvar WaveBitOpKind_Or = 1;
defvar WaveBitOpKind_Xor = 2;
+defvar WaveMultiPrefixOpKind_Sum = 0;
defvar WaveMultiPrefixOpKind_And = 1;
defvar WaveMultiPrefixOpKind_Xor = 3;
defvar WaveMultiPrefixOpKind_Product = 4;
@@ -1593,6 +1594,20 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> {
IntrinArgIndex<3>, IntrinArgIndex<4>,
IntrinArgI8<WaveMultiPrefixOpKind_Product>,
IntrinArgI8<SignedOpKind_Unsigned>
+ ]>,
+ IntrinSelect<int_dx_wave_multi_prefix_sum,
+ [
+ IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>,
+ IntrinArgIndex<3>, IntrinArgIndex<4>,
+ IntrinArgI8<WaveMultiPrefixOpKind_Sum>,
+ IntrinArgI8<SignedOpKind_Signed>
+ ]>,
+ IntrinSelect<int_dx_wave_multi_prefix_usum,
+ [
+ IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>,
+ IntrinArgIndex<3>, IntrinArgIndex<4>,
+ IntrinArgI8<WaveMultiPrefixOpKind_Sum>,
+ IntrinArgI8<SignedOpKind_Unsigned>
]>
];
let arguments = [
diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
index ccfe6e9c947c0..cbbc17de32e96 100644
--- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
+++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
@@ -116,6 +116,8 @@ static bool checkWaveOps(Intrinsic::ID IID) {
case Intrinsic::dx_wave_multi_prefix_bit_count:
case Intrinsic::dx_wave_multi_prefix_product:
case Intrinsic::dx_wave_multi_prefix_uproduct:
+ case Intrinsic::dx_wave_multi_prefix_sum:
+ case Intrinsic::dx_wave_multi_prefix_usum:
// Quad Op Variants
case Intrinsic::dx_quad_read_across_x:
case Intrinsic::dx_quad_read_across_y:
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index 320156ee3d153..8bf0de1c60859 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -24,6 +24,8 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg(
case Intrinsic::dx_wave_multi_prefix_bit_xor:
case Intrinsic::dx_wave_multi_prefix_product:
case Intrinsic::dx_wave_multi_prefix_uproduct:
+ case Intrinsic::dx_wave_multi_prefix_sum:
+ case Intrinsic::dx_wave_multi_prefix_usum:
return ScalarOpdIdx >= 1 && ScalarOpdIdx <= 4;
default:
return false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index b72d5517e5ce9..c61ed5c02d8a8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -5846,6 +5846,14 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
: SPIRV::OpGroupNonUniformIMul,
"WaveMultiPrefixProduct");
}
+ case Intrinsic::spv_subgroup_multi_prefix_sum: {
+ bool IsFloatTy =
+ GR.isScalarOrVectorOfType(I.getOperand(2).getReg(), SPIRV::OpTypeFloat);
+ return selectWaveMultiPrefixOp(ResVReg, ResType, I,
+ IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
+ : SPIRV::OpGroupNonUniformIAdd,
+ "WaveMultiPrefixSum");
+ }
case Intrinsic::spv_wave_prefix_product:
return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
case Intrinsic::spv_quad_read_across_x: {
diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
index 5ee508ab5c49c..f045f0f85b95a 100644
--- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
+++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
@@ -105,6 +105,20 @@ define i32 @wave_multi_prefix_uproduct(i32 %x) {
ret i32 %ret
}
+define float @wave_multi_prefix_sum(float %x) {
+ ; CHECK: Function wave_multi_prefix_sum : [[WAVE_FLAG]]
+ %ret = call float @llvm.dx.wave.multi.prefix.sum.f32(
+ float %x, i32 1, i32 2, i32 4, i32 8)
+ ret float %ret
+}
+
+define i32 @wave_multi_prefix_usum(i32 %x) {
+ ; CHECK: Function wave_multi_prefix_usum : [[WAVE_FLAG]]
+ %ret = cal...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/238175
More information about the llvm-branch-commits
mailing list