[llvm-branch-commits] [clang] [llvm] [HLSL] Add WaveMultiPrefixBitXor (PR #238097)
Joshua Batista via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 2 13:01:12 PST 2026
https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/238097
Adds WaveMultiPrefixBitXor for Shader Model 6.5 and newer. Adds Clang checks and code generation, DXIL lowering, and SPIR-V lowering.
Uses `llvm.spv.subgroup.multi.prefix.bit.xor` for SPIR-V. Accepts either partition extension and prefers EXT when both EXT and NV are enabled. Reuses the BitAnd mask checks and SPIR-V lowering code.
This is the second PR in the five-PR WaveMultiPrefix stack. It builds on #238083.
The existing SPIR-V limit for long-vector function parameters remains. The backend test uses memory loads and stores for a long vector.
Fixes #99210.
>From e6d56733761e1bc60557f2ef60578340ef8fa7ac Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Mon, 2 Nov 2026 13:00:31 -0800
Subject: [PATCH] first attempt
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
Copilot-Session: 40997c2f-5aaa-41c6-8d1f-3e2e9ada315f
---
clang/include/clang/Basic/Builtins.td | 6 ++
clang/include/clang/Basic/HLSLIntrinsics.td | 10 ++
clang/lib/CodeGen/CGHLSLBuiltins.cpp | 11 ++-
clang/lib/CodeGen/CGHLSLRuntime.h | 3 +
clang/lib/Sema/SemaHLSL.cpp | 3 +-
.../builtins/WaveMultiPrefixBitXor.hlsl | 97 ++++++++++++++++++
.../WaveMultiPrefixBitXor-errors.hlsl | 64 ++++++++++++
llvm/include/llvm/IR/IntrinsicsDirectX.td | 7 ++
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 7 ++
llvm/lib/Target/DirectX/DXIL.td | 8 ++
llvm/lib/Target/DirectX/DXILShaderFlags.cpp | 1 +
.../DirectX/DirectXTargetTransformInfo.cpp | 1 +
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 26 +++--
.../CodeGen/DirectX/ShaderFlags/wave-ops.ll | 7 ++
.../CodeGen/DirectX/WaveMultiPrefixBitXor.ll | 48 +++++++++
.../DirectX/WaveMultiPrefixBitXor_errors.ll | 14 +++
.../hlsl-intrinsics/WaveMultiPrefixBitXor.ll | 98 +++++++++++++++++++
17 files changed, 398 insertions(+), 13 deletions(-)
create mode 100644 clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl
create mode 100644 clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl
create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll
create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll
create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 5ae3edcc6601c..ab4b7f4b1187a 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5795,6 +5795,12 @@ def HLSLWaveMultiPrefixBitAnd : LangBuiltin<"HLSL_LANG"> {
let Prototype = "void(...)";
}
+def HLSLWaveMultiPrefixBitXor : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_wave_multi_prefix_bit_xor"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "void(...)";
+}
+
def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_wave_prefix_product"];
let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td
index 902a31d8278b3..300d46a7fd324 100644
--- a/clang/include/clang/Basic/HLSLIntrinsics.td
+++ b/clang/include/clang/Basic/HLSLIntrinsics.td
@@ -1893,6 +1893,16 @@ def hlsl_wave_multi_prefix_bit_and :
let Availability = SM6_5;
}
+def hlsl_wave_multi_prefix_bit_xor :
+ HLSLTwoArgBuiltin<"WaveMultiPrefixBitXor",
+ "__builtin_hlsl_wave_multi_prefix_bit_xor"> {
+ let Args = [Varying, VectorType<UIntTy, 4>];
+ let VaryingTypes = AllIntTypes;
+ let VaryingLongVector = 1;
+ let IsConvergent = 1;
+ let Availability = SM6_5;
+}
+
// Returns the count of bits set to 1 on prior lanes.
def hlsl_wave_prefix_count_bits :
HLSLBuiltin<"WavePrefixCountBits",
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index d4f5f47d9b24e..3afc8a0668f27 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -1721,15 +1721,20 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
return EmitIntrinsicCall(IID, {OpExpr->getType()}, ArrayRef{OpExpr},
"hlsl.wave.prefix.sum");
}
- case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: {
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: {
Value *ValueOp = EmitScalarExpr(E->getArg(0));
Value *Mask = EmitScalarExpr(E->getArg(1));
SmallVector<Value *, 5> Args{ValueOp};
for (unsigned I = 0; I != 4; ++I)
Args.push_back(Builder.CreateExtractElement(Mask, I));
+ if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and)
+ return EmitIntrinsicCall(
+ CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
+ {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.and");
return EmitIntrinsicCall(
- CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
- {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.and");
+ CGM.getHLSLRuntime().getWaveMultiPrefixBitXorIntrinsic(),
+ {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.xor");
}
case Builtin::BI__builtin_hlsl_wave_prefix_product: {
Value *OpExpr = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index 89b52f287d9ed..71d8255ab3279 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -152,6 +152,9 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixBitAnd,
wave_multi_prefix_bit_and,
subgroup_multi_prefix_bit_and)
+ GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixBitXor,
+ wave_multi_prefix_bit_xor,
+ subgroup_multi_prefix_bit_xor)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 6147a0446a838..d0da34ef79f89 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -5377,7 +5377,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
TheCall->setType(ArgTyExpr);
break;
}
- case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: {
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
+ case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: {
if (SemaRef.checkArgCount(TheCall, 2))
return true;
diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl
new file mode 100644
index 0000000000000..b903fe8a168cc
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl
@@ -0,0 +1,97 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | \
+// RUN: FileCheck %s -DINTRINSIC=dx.wave.multi.prefix.bit.xor
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple spirv-pc-vulkan-library %s -emit-llvm \
+// RUN: -disable-llvm-passes -o - | \
+// RUN: FileCheck %s -DINTRINSIC=spv.subgroup.multi.prefix.bit.xor
+
+// CHECK-LABEL: test_uint
+uint test_uint(uint value, uint4 mask) {
+ // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry()
+ // CHECK: %[[MASK:.*]] = load <4 x i32>
+ // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0
+ // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1
+ // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2
+ // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3
+ // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ]
+ // CHECK: ret i32 %[[RET]]
+ return WaveMultiPrefixBitXor(value, mask);
+}
+
+// CHECK: declare i32 @llvm.[[INTRINSIC]].i32(i32, i32, i32, i32, i32) #[[ATTR:[0-9]+]]
+
+// CHECK-LABEL: test_int
+int test_int(int value) {
+ // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i32 %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int16
+int16_t test_int16(int16_t value) {
+ // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i16 %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint16
+uint16_t test_uint16(uint16_t value) {
+ // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i16 %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int64
+int64_t test_int64(int64_t value) {
+ // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i64 %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint64
+uint64_t test_uint64(uint64_t value) {
+ // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret i64 %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint2
+uint2 test_uint2(uint2 value) {
+ // CHECK: %[[RET:.*]] = call <2 x i32> @llvm.[[INTRINSIC]].v2i32(<2 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <2 x i32> %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int3
+int3 test_int3(int3 value) {
+ // CHECK: %[[RET:.*]] = call <3 x i32> @llvm.[[INTRINSIC]].v3i32(<3 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <3 x i32> %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint64_t4
+uint64_t4 test_uint64_t4(uint64_t4 value) {
+ // CHECK: %[[RET:.*]] = call <4 x i64> @llvm.[[INTRINSIC]].v4i64(<4 x i64> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <4 x i64> %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint5
+vector<uint, 5> test_uint5(vector<uint, 5> value) {
+ // CHECK: %[[RET:.*]] = call <5 x i32> @llvm.[[INTRINSIC]].v5i32(<5 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <5 x i32> %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int2x3
+int2x3 test_int2x3(int2x3 value) {
+ // CHECK: %[[RET:.*]] = call <6 x i32> @llvm.[[INTRINSIC]].v6i32(<6 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8)
+ // CHECK: ret <6 x i32> %[[RET]]
+ return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK: attributes #[[ATTR]] = { {{.*}}convergent {{.*}} }
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl
new file mode 100644
index 0000000000000..6c1365c92accd
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN: -fnative-half-type -fnative-int16-type \
+// RUN: -triple spirv-pc-vulkan-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \
+// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s
+
+#ifdef TEST_AVAILABILITY
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main() {
+ (void)WaveMultiPrefixBitXor(1u, uint4(1, 2, 4, 8));
+ // availability-error at -1 {{only available on Shader Model 6.5 or newer}}
+ // availability-note@*:* {{introduced in Shader Model 6.5}}
+}
+#else
+void test_arg_count(uint value, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value);
+ // expected-error at -1 {{too few arguments to function call, expected 2}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value, mask, mask);
+ // expected-error at -1 {{too many arguments to function call, expected 2}}
+}
+
+struct S { uint value; };
+
+void test_value_types(float f, float2 fv, bool b, bool2 bv, S s,
+ float2x2 fm, bool2x2 bm, uint4 mask) {
+ __builtin_hlsl_wave_multi_prefix_bit_xor(f, mask);
+ // expected-error at -1 {{1st argument must be a scalar or vector of integer}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(fv, mask);
+ // expected-error at -1 {{1st argument must be a scalar or vector of integer}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(b, mask);
+ // expected-error at -1 {{invalid operand of type 'bool'}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(bv, mask);
+ // expected-error at -1 {{invalid operand of type 'bool2'}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(s, mask);
+ // expected-error at -1 {{invalid operand of type 'S'}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(fm, mask);
+ // expected-error at -1 {{1st argument must be a matrix integer type}}
+ // expected-error at -2 {{cannot pass object of non-trivial type 'float2x2'}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(bm, mask);
+ // expected-error at -1 {{1st argument must be a matrix integer type}}
+ // expected-error at -2 {{cannot pass object of non-trivial type 'bool2x2'}}
+}
+
+void test_mask_types(uint value, uint scalar, uint3 short_mask,
+ int4 signed_mask, float4 float_mask,
+ uint64_t4 wide_mask, S struct_mask) {
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value, scalar);
+ // expected-error at -1 {{passing 'uint' (aka 'unsigned int')}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value, short_mask);
+ // expected-error at -1 {{passing 'uint3' (aka 'vector<uint, 3>')}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value, signed_mask);
+ // expected-error at -1 {{passing 'int4' (aka 'vector<int, 4>')}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value, float_mask);
+ // expected-error at -1 {{passing 'float4' (aka 'vector<float, 4>')}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value, wide_mask);
+ // expected-error at -1 {{passing 'uint64_t4'}}
+ __builtin_hlsl_wave_multi_prefix_bit_xor(value, struct_mask);
+ // expected-error at -1 {{passing 'S' to parameter of incompatible type}}
+}
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 4f23541e7eeed..c292a0a46b2ae 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -481,6 +481,13 @@ let TargetPrefix = "dx" in {
[IntrConvergent, IntrNoMem,
IntrTriviallyScalarizable]>;
+ def int_dx_wave_multi_prefix_bit_xor
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem,
+ IntrTriviallyScalarizable]>;
+
def int_dx_wave_prefix_usum
: DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>],
[IntrConvergent, IntrNoMem,
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 2b1127445f799..0c6dc262c3b09 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -390,6 +390,13 @@ let TargetPrefix = "spv" in {
[IntrConvergent, IntrNoMem,
IntrTriviallyScalarizable]>;
+ def int_spv_subgroup_multi_prefix_bit_xor
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrNoMem,
+ IntrTriviallyScalarizable]>;
+
def int_spv_wave_prefix_product
: DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>],
[IntrConvergent, IntrNoMem,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index c2165c9664a3d..cf3d6404f6210 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -325,6 +325,7 @@ defvar WaveBitOpKind_Or = 1;
defvar WaveBitOpKind_Xor = 2;
defvar WaveMultiPrefixOpKind_And = 1;
+defvar WaveMultiPrefixOpKind_Xor = 3;
defvar SignedOpKind_Signed = 0;
defvar SignedOpKind_Unsigned = 1;
@@ -1570,6 +1571,13 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> {
IntrinArgIndex<3>, IntrinArgIndex<4>,
IntrinArgI8<WaveMultiPrefixOpKind_And>,
IntrinArgI8<SignedOpKind_Signed>
+ ]>,
+ IntrinSelect<int_dx_wave_multi_prefix_bit_xor,
+ [
+ IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>,
+ IntrinArgIndex<3>, IntrinArgIndex<4>,
+ IntrinArgI8<WaveMultiPrefixOpKind_Xor>,
+ IntrinArgI8<SignedOpKind_Signed>
]>
];
let arguments = [
diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
index 1c27d16e05979..1f37844dc5f19 100644
--- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
+++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
@@ -112,6 +112,7 @@ static bool checkWaveOps(Intrinsic::ID IID) {
case Intrinsic::dx_wave_prefix_product:
case Intrinsic::dx_wave_prefix_uproduct:
case Intrinsic::dx_wave_multi_prefix_bit_and:
+ case Intrinsic::dx_wave_multi_prefix_bit_xor:
// Quad Op Variants
case Intrinsic::dx_quad_read_across_x:
case Intrinsic::dx_quad_read_across_y:
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index 6fa1d27533442..00ef806735eff 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -21,6 +21,7 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg(
case Intrinsic::dx_wave_readlane:
return ScalarOpdIdx == 1;
case Intrinsic::dx_wave_multi_prefix_bit_and:
+ case Intrinsic::dx_wave_multi_prefix_bit_xor:
return ScalarOpdIdx >= 1 && ScalarOpdIdx <= 4;
default:
return false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index a1ae8d2edce95..ec9e548d319b2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -322,8 +322,9 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
- bool selectWaveMultiPrefixBitAnd(Register ResVReg, SPIRVTypeInst ResType,
- MachineInstr &I) const;
+ bool selectWaveMultiPrefixOp(Register ResVReg, SPIRVTypeInst ResType,
+ MachineInstr &I, unsigned Opcode,
+ StringRef BuiltinName) const;
bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
unsigned Direction) const;
@@ -4014,15 +4015,17 @@ bool SPIRVInstructionSelector::selectWaveExclusiveScan(
return true;
}
-bool SPIRVInstructionSelector::selectWaveMultiPrefixBitAnd(
- Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
+bool SPIRVInstructionSelector::selectWaveMultiPrefixOp(
+ Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, unsigned Opcode,
+ StringRef BuiltinName) const {
if (!STI.canUseExtension(
SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned) &&
!STI.canUseExtension(
SPIRV::Extension::SPV_NV_shader_subgroup_partitioned))
return diagnoseUnsupported(
- I, "WaveMultiPrefixBitAnd requires SPV_EXT_shader_subgroup_partitioned "
- "or SPV_NV_shader_subgroup_partitioned");
+ I, Twine(BuiltinName) +
+ " requires SPV_EXT_shader_subgroup_partitioned or "
+ "SPV_NV_shader_subgroup_partitioned");
assert(I.getNumOperands() == 7);
SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
@@ -4037,8 +4040,7 @@ bool SPIRVInstructionSelector::selectWaveMultiPrefixBitAnd(
MaskMIB.addUse(I.getOperand(Op).getReg());
MaskMIB.constrainAllUses(TII, TRI, RBI);
- BuildMI(*I.getParent(), I, I.getDebugLoc(),
- TII.get(SPIRV::OpGroupNonUniformBitwiseAnd))
+ BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
.addDef(ResVReg)
.addUse(GR.getSPIRVTypeID(ResType))
.addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
@@ -5829,7 +5831,13 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
case Intrinsic::spv_wave_prefix_sum:
return selectWaveExclusiveScanSum(ResVReg, ResType, I);
case Intrinsic::spv_subgroup_multi_prefix_bit_and:
- return selectWaveMultiPrefixBitAnd(ResVReg, ResType, I);
+ return selectWaveMultiPrefixOp(ResVReg, ResType, I,
+ SPIRV::OpGroupNonUniformBitwiseAnd,
+ "WaveMultiPrefixBitAnd");
+ case Intrinsic::spv_subgroup_multi_prefix_bit_xor:
+ return selectWaveMultiPrefixOp(ResVReg, ResType, I,
+ SPIRV::OpGroupNonUniformBitwiseXor,
+ "WaveMultiPrefixBitXor");
case Intrinsic::spv_wave_prefix_product:
return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
case Intrinsic::spv_quad_read_across_x: {
diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
index 55dd18c81f420..833b5fee3d946 100644
--- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
+++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
@@ -77,6 +77,13 @@ define i32 @wave_multi_prefix_bit_and(i32 %x) {
ret i32 %ret
}
+define i32 @wave_multi_prefix_bit_xor(i32 %x) {
+ ; CHECK: Function wave_multi_prefix_bit_xor : [[WAVE_FLAG]]
+ %ret = call i32 @llvm.dx.wave.multi.prefix.bit.xor.i32(
+ i32 %x, i32 1, i32 2, i32 4, i32 8)
+ ret i32 %ret
+}
+
define noundef i1 @wave_all_equal(i1 %x) {
entry:
; CHECK: Function wave_all_equal : [[WAVE_FLAG]]
diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll
new file mode 100644
index 0000000000000..d5475630a6f7d
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll
@@ -0,0 +1,48 @@
+; RUN: opt -S -scalarizer -dxil-op-lower \
+; RUN: -mtriple=dxil-pc-shadermodel6.5-library < %s | FileCheck %s
+
+; CHECK-LABEL: define i16 @test_i16(
+; CHECK: call i16 @dx.op.waveMultiPrefixOp.i16(i32 166, i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0)
+define i16 @test_i16(i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) {
+ %ret = call i16 @llvm.dx.wave.multi.prefix.bit.xor.i16(
+ i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+ ret i16 %ret
+}
+
+; CHECK-LABEL: define i32 @test_i32(
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0)
+define i32 @test_i32(i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) {
+ %ret = call i32 @llvm.dx.wave.multi.prefix.bit.xor.i32(
+ i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+ ret i32 %ret
+}
+
+; CHECK-LABEL: define i64 @test_i64(
+; CHECK: call i64 @dx.op.waveMultiPrefixOp.i64(i32 166, i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0)
+define i64 @test_i64(i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) {
+ %ret = call i64 @llvm.dx.wave.multi.prefix.bit.xor.i64(
+ i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+ ret i64 %ret
+}
+
+; CHECK-LABEL: define <2 x i32> @test_vector(
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i0, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0)
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i1, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0)
+define <2 x i32> @test_vector(<2 x i32> %value, i32 %m0, i32 %m1,
+ i32 %m2, i32 %m3) {
+ %ret = call <2 x i32> @llvm.dx.wave.multi.prefix.bit.xor.v2i32(
+ <2 x i32> %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+ ret <2 x i32> %ret
+}
+
+; CHECK-LABEL: define <5 x i32> @test_long_vector(
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i0, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0)
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i1, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0)
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i2, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0)
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i3, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0)
+; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i4, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0)
+define <5 x i32> @test_long_vector(<5 x i32> %value) {
+ %ret = call <5 x i32> @llvm.dx.wave.multi.prefix.bit.xor.v5i32(
+ <5 x i32> %value, i32 1, i32 2, i32 4, i32 8)
+ ret <5 x i32> %ret
+}
diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll
new file mode 100644
index 0000000000000..757c64940e02e
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll
@@ -0,0 +1,14 @@
+; RUN: not opt -S -dxil-op-lower \
+; RUN: -mtriple=dxil-pc-shadermodel6.4-library %s 2>&1 | \
+; RUN: FileCheck %s --check-prefix=VERSION
+; RUN: not opt -S -dxil-op-lower \
+; RUN: -mtriple=dxil-pc-shadermodel6.5-library %s 2>&1 | \
+; RUN: FileCheck %s --check-prefix=TYPE
+
+; VERSION: Cannot create WaveMultiPrefixOp operation: No valid overloads for DXIL version 1.4
+; TYPE: Cannot create WaveMultiPrefixOp operation: Invalid overload type
+define i8 @test_i8(i8 %value) {
+ %ret = call i8 @llvm.dx.wave.multi.prefix.bit.xor.i8(
+ i8 %value, i32 1, i32 2, i32 4, i32 8)
+ ret i8 %ret
+}
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll
new file mode 100644
index 0000000000000..876fb0c25a1d6
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll
@@ -0,0 +1,98 @@
+; DEFINE: %{nv} = SPV_NV_shader_subgroup_partitioned
+; DEFINE: %{ext} = SPV_EXT_shader_subgroup_partitioned
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \
+; RUN: --spirv-ext=+%{nv} %s -o - | FileCheck %s -DEXTENSION=NV \
+; RUN: --implicit-check-not=%{ext}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \
+; RUN: --spirv-ext=+%{ext} %s -o - | FileCheck %s -DEXTENSION=EXT \
+; RUN: --implicit-check-not=%{nv}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \
+; RUN: --spirv-ext=+%{nv},+%{ext} %s -o - | FileCheck %s -DEXTENSION=EXT \
+; RUN: --implicit-check-not=%{nv}
+; RUN: not llc -O0 -mtriple=spirv-vulkan-unknown %s -o - 2>&1 | \
+; RUN: FileCheck %s --check-prefix=NO-EXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \
+; RUN: --spirv-ext=+%{nv} %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \
+; RUN: --spirv-ext=+%{ext} %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \
+; RUN: --spirv-ext=+%{nv},+%{ext} %s -o - \
+; RUN: -filetype=obj | spirv-val %}
+
+; NO-EXT: WaveMultiPrefixBitXor requires SPV_EXT_shader_subgroup_partitioned or SPV_NV_shader_subgroup_partitioned
+
+; CHECK-DAG: OpCapability GroupNonUniformPartitionedNV
+; CHECK-DAG: OpExtension "SPV_[[EXTENSION]]_shader_subgroup_partitioned"
+; CHECK-DAG: %[[#I16:]] = OpTypeInt 16 0
+; CHECK-DAG: %[[#I32:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#I64:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#V2I32:]] = OpTypeVector %[[#I32]] 2
+; CHECK-DAG: %[[#V4I32:]] = OpTypeVector %[[#I32]] 4
+; CHECK-DAG: %[[#SCOPE:]] = OpConstant %[[#I32]] 3
+; CHECK-DAG: %[[#C1:]] = OpConstant %[[#I32]] 1
+; CHECK-DAG: %[[#C2:]] = OpConstant %[[#I32]] 2
+; CHECK-DAG: %[[#C4:]] = OpConstant %[[#I32]] 4
+; CHECK-DAG: %[[#C8:]] = OpConstant %[[#I32]] 8
+
+ at wide_i32 = internal addrspace(10) global [5 x i32] zeroinitializer
+
+; CHECK-LABEL: Begin function test_i32
+; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I32]]
+; CHECK: %[[#M0:]] = OpFunctionParameter %[[#I32]]
+; CHECK: %[[#M1:]] = OpFunctionParameter %[[#I32]]
+; CHECK: %[[#M2:]] = OpFunctionParameter %[[#I32]]
+; CHECK: %[[#M3:]] = OpFunctionParameter %[[#I32]]
+; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#M0]] %[[#M1]] %[[#M2]] %[[#M3]]
+; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]]
+; CHECK: OpReturnValue %[[#RET]]
+define i32 @test_i32(i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) {
+ %ret = call i32 @llvm.spv.subgroup.multi.prefix.bit.xor.i32(
+ i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3)
+ ret i32 %ret
+}
+
+; CHECK-LABEL: Begin function test_i16
+; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I16]]
+; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]]
+; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#I16]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]]
+; CHECK: OpReturnValue %[[#RET]]
+define i16 @test_i16(i16 %value) {
+ %ret = call i16 @llvm.spv.subgroup.multi.prefix.bit.xor.i16(
+ i16 %value, i32 1, i32 2, i32 4, i32 8)
+ ret i16 %ret
+}
+
+; CHECK-LABEL: Begin function test_i64
+; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I64]]
+; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]]
+; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#I64]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]]
+; CHECK: OpReturnValue %[[#RET]]
+define i64 @test_i64(i64 %value) {
+ %ret = call i64 @llvm.spv.subgroup.multi.prefix.bit.xor.i64(
+ i64 %value, i32 1, i32 2, i32 4, i32 8)
+ ret i64 %ret
+}
+
+; CHECK-LABEL: Begin function test_vector
+; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#V2I32]]
+; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]]
+; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#V2I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]]
+; CHECK: OpReturnValue %[[#RET]]
+define <2 x i32> @test_vector(<2 x i32> %value) {
+ %ret = call <2 x i32> @llvm.spv.subgroup.multi.prefix.bit.xor.v2i32(
+ <2 x i32> %value, i32 1, i32 2, i32 4, i32 8)
+ ret <2 x i32> %ret
+}
+
+; CHECK-LABEL: Begin function test_long_vector
+; CHECK: %[[#MASK0:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]]
+; CHECK: OpGroupNonUniformBitwiseXor %[[#V4I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK0]]
+; CHECK: %[[#MASK1:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]]
+; CHECK: OpGroupNonUniformBitwiseXor %[[#I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK1]]
+define void @test_long_vector() {
+ %value = load <5 x i32>, ptr addrspace(10) @wide_i32
+ %ret = call <5 x i32> @llvm.spv.subgroup.multi.prefix.bit.xor.v5i32(
+ <5 x i32> %value, i32 1, i32 2, i32 4, i32 8)
+ store <5 x i32> %ret, ptr addrspace(10) @wide_i32
+ ret void
+}
More information about the llvm-branch-commits
mailing list