[llvm] d8f71b1 - [SPIR-V] Add support for SPV_INTEL_masked_gather_scatter extension (#185418)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 11 05:14:41 PDT 2026
Author: Arseniy Obolenskiy
Date: 2026-03-11T13:14:36+01:00
New Revision: d8f71b16387571c1613c0192ba4eb001f7c0791d
URL: https://github.com/llvm/llvm-project/commit/d8f71b16387571c1613c0192ba4eb001f7c0791d
DIFF: https://github.com/llvm/llvm-project/commit/d8f71b16387571c1613c0192ba4eb001f7c0791d.diff
LOG: [SPIR-V] Add support for SPV_INTEL_masked_gather_scatter extension (#185418)
Fixes the first bullet in #184638
Corresponding patch to add support for vector operands in
OpConvertPtrToU/OpConvertUToPtr operations in spirv-val:
https://github.com/KhronosGroup/SPIRV-Tools/pull/6575
SPIR-V extension reference used:
https://github.com/KhronosGroup/SPIRV-Registry/blob/278044a51fee280bfc91322cdb55b51357db5cb8/extensions/INTEL/SPV_INTEL_masked_gather_scatter.asciidoc
Added:
llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-no-extension.ll
llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-scatter.ll
llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-scatter-no-extension.ll
llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-no-extension.ll
llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-ptrtoint.ll
Modified:
llvm/include/llvm/IR/IntrinsicsSPIRV.td
llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.cpp
llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 6777ddef292d7..ead6d18124381 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -44,6 +44,14 @@ let TargetPrefix = "spv" in {
def int_spv_undef : Intrinsic<[llvm_i32_ty], []>;
def int_spv_inline_asm : Intrinsic<[], [llvm_metadata_ty, llvm_metadata_ty, llvm_vararg_ty]>;
+ // Masked Gather/Scatter (SPV_INTEL_masked_gather_scatter)
+ def int_spv_masked_gather : Intrinsic<[llvm_any_ty],
+ [llvm_any_ty, llvm_i32_ty, llvm_any_ty, llvm_any_ty],
+ [IntrReadMem, IntrWillReturn, ImmArg<ArgIndex<1>>]>;
+ def int_spv_masked_scatter : Intrinsic<[],
+ [llvm_any_ty, llvm_any_ty, llvm_i32_ty, llvm_any_ty],
+ [IntrWriteMem, IntrWillReturn, ImmArg<ArgIndex<2>>]>;
+
// Expect, Assume Intrinsics
def int_spv_assume : Intrinsic<[], [llvm_i1_ty]>;
def int_spv_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index 33e1b52b724e6..734a03ff60141 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -86,6 +86,8 @@ static const StringMap<SPIRV::Extension::Extension> SPIRVExtensionMap = {
SPIRV::Extension::Extension::SPV_INTEL_memory_access_aliasing},
{"SPV_INTEL_joint_matrix",
SPIRV::Extension::Extension::SPV_INTEL_joint_matrix},
+ {"SPV_INTEL_masked_gather_scatter",
+ SPIRV::Extension::Extension::SPV_INTEL_masked_gather_scatter},
{"SPV_KHR_16bit_storage",
SPIRV::Extension::Extension::SPV_KHR_16bit_storage},
{"SPV_KHR_device_group", SPIRV::Extension::Extension::SPV_KHR_device_group},
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index c84f41dada005..73d05f4dc574a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -299,6 +299,8 @@ class SPIRVEmitIntrinsics
bool processFunctionPointers(Module &M);
void parseFunDeclarations(Module &M);
void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
+ bool processMaskedMemIntrinsic(IntrinsicInst &I);
+ bool convertMaskedMemIntrinsics(Module &M);
void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
@@ -3281,9 +3283,101 @@ void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) {
}
}
+bool SPIRVEmitIntrinsics::processMaskedMemIntrinsic(IntrinsicInst &I) {
+ const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(*I.getFunction());
+
+ if (I.getIntrinsicID() == Intrinsic::masked_gather) {
+ if (!ST.canUseExtension(
+ SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
+ I.getContext().emitError(
+ &I, "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
+ "extension");
+ // Replace with poison to allow compilation to continue and report error.
+ I.replaceAllUsesWith(PoisonValue::get(I.getType()));
+ I.eraseFromParent();
+ return true;
+ }
+
+ IRBuilder<> B(&I);
+
+ Value *Ptrs = I.getArgOperand(0);
+ Value *Mask = I.getArgOperand(1);
+ Value *Passthru = I.getArgOperand(2);
+
+ // Alignment is stored as a parameter attribute, not as a regular parameter
+ uint32_t Alignment = I.getParamAlign(0).valueOrOne().value();
+
+ SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(Alignment), Mask,
+ Passthru};
+ SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
+ Mask->getType(), Passthru->getType()};
+
+ auto *NewI = B.CreateIntrinsic(Intrinsic::spv_masked_gather, Types, Args);
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ return true;
+ }
+
+ if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
+ if (!ST.canUseExtension(
+ SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
+ I.getContext().emitError(
+ &I, "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
+ "extension");
+ // Erase the intrinsic to allow compilation to continue and report error.
+ I.eraseFromParent();
+ return true;
+ }
+
+ IRBuilder<> B(&I);
+
+ Value *Values = I.getArgOperand(0);
+ Value *Ptrs = I.getArgOperand(1);
+ Value *Mask = I.getArgOperand(2);
+
+ // Alignment is stored as a parameter attribute on the ptrs parameter (arg
+ // 1)
+ uint32_t Alignment = I.getParamAlign(1).valueOrOne().value();
+
+ SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(Alignment), Mask};
+ SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
+ Mask->getType()};
+
+ B.CreateIntrinsic(Intrinsic::spv_masked_scatter, Types, Args);
+ I.eraseFromParent();
+ return true;
+ }
+
+ return false;
+}
+
+bool SPIRVEmitIntrinsics::convertMaskedMemIntrinsics(Module &M) {
+ bool Changed = false;
+
+ for (Function &F : make_early_inc_range(M)) {
+ if (!F.isIntrinsic())
+ continue;
+ Intrinsic::ID IID = F.getIntrinsicID();
+ if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
+ continue;
+
+ for (User *U : make_early_inc_range(F.users())) {
+ if (auto *II = dyn_cast<IntrinsicInst>(U))
+ Changed |= processMaskedMemIntrinsic(*II);
+ }
+
+ if (F.use_empty())
+ F.eraseFromParent();
+ }
+
+ return Changed;
+}
+
bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
bool Changed = false;
+ Changed |= convertMaskedMemIntrinsics(M);
+
parseFunDeclarations(M);
insertConstantsForFPFastMathDefault(M);
GVUsers.init(M);
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 9a85634c82626..cf4ab00a4f3b3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -318,17 +318,31 @@ SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVTypeInst ElemType,
auto EleOpc = ElemType->getOpcode();
(void)EleOpc;
assert(NumElems >= 2 && "SPIR-V OpTypeVector requires at least 2 components");
- assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||
- EleOpc == SPIRV::OpTypeBool) &&
- "Invalid vector element type");
- return createConstOrTypeAtFunctionEntry(MIRBuilder, [&](MachineIRBuilder
- &MIRBuilder) {
- return MIRBuilder.buildInstr(SPIRV::OpTypeVector)
- .addDef(createTypeVReg(MIRBuilder))
- .addUse(getSPIRVTypeID(ElemType))
- .addImm(NumElems);
- });
+ if (EleOpc == SPIRV::OpTypePointer) {
+ if (!cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget())
+ .canUseExtension(
+ SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
+ const Function &F = MIRBuilder.getMF().getFunction();
+ F.getContext().diagnose(DiagnosticInfoUnsupported(
+ F,
+ "Vector of pointers requires SPV_INTEL_masked_gather_scatter "
+ "extension",
+ DebugLoc(), DS_Error));
+ }
+ } else {
+ assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||
+ EleOpc == SPIRV::OpTypeBool) &&
+ "Invalid vector element type");
+ }
+
+ return createConstOrTypeAtFunctionEntry(
+ MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+ return MIRBuilder.buildInstr(SPIRV::OpTypeVector)
+ .addDef(createTypeVReg(MIRBuilder))
+ .addUse(getSPIRVTypeID(ElemType))
+ .addImm(NumElems);
+ });
}
Register SPIRVGlobalRegistry::getOrCreateConstFP(APFloat Val, MachineInstr &I,
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index d2f81bc30e949..819cdd6107d0d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -1131,3 +1131,9 @@ def OpFixedLogALTERA: Op<5932, (outs ID:$res), (ins TYPE:$result_type, ID:$input
"$res = OpFixedLogALTERA $result_type $input $sign $l $rl $q $o">;
def OpFixedExpALTERA: Op<5933, (outs ID:$res), (ins TYPE:$result_type, ID:$input, i32imm:$sign, i32imm:$l, i32imm:$rl, i32imm:$q, i32imm:$o),
"$res = OpFixedExpALTERA $result_type $input $sign $l $rl $q $o">;
+
+//SPV_INTEL_masked_gather_scatter
+def OpMaskedGatherINTEL: Op<6428, (outs ID:$res), (ins TYPE:$resType, ID:$ptrs, ID:$alignment, ID:$mask, ID:$fillEmpty),
+ "$res = OpMaskedGatherINTEL $resType $ptrs $alignment $mask $fillEmpty">;
+def OpMaskedScatterINTEL: Op<6429, (outs), (ins ID:$ptrs, ID:$alignment, ID:$mask, ID:$values),
+ "OpMaskedScatterINTEL $ptrs $alignment $mask $values">;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 002ee0d6e13a8..fcae23432017d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -29,6 +29,7 @@
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
@@ -306,6 +307,12 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectGEP(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
+ bool selectMaskedGather(Register ResVReg, SPIRVTypeInst ResType,
+ MachineInstr &I) const;
+ bool selectMaskedScatter(MachineInstr &I) const;
+
+ bool diagnoseUnsupported(const MachineInstr &I, const Twine &Msg) const;
+
bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType,
@@ -1733,6 +1740,69 @@ bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
return true;
}
+bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg,
+ SPIRVTypeInst ResType,
+ MachineInstr &I) const {
+ assert(I.getNumExplicitDefs() == 1 && "Expected single def for gather");
+ // Operand indices:
+ // 0: result (def)
+ // 1: intrinsic ID
+ // 2: vector of pointers
+ // 3: alignment (i32 immediate)
+ // 4: mask (vector of i1)
+ // 5: passthru/fill value
+ Register PtrsReg = I.getOperand(2).getReg();
+ uint32_t Alignment = I.getOperand(3).getImm();
+ Register MaskReg = I.getOperand(4).getReg();
+ Register PassthruReg = I.getOperand(5).getReg();
+ Register AlignmentReg = buildI32Constant(Alignment, I);
+
+ MachineBasicBlock &BB = *I.getParent();
+ auto MIB =
+ BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedGatherINTEL))
+ .addDef(ResVReg)
+ .addUse(GR.getSPIRVTypeID(ResType))
+ .addUse(PtrsReg)
+ .addUse(AlignmentReg)
+ .addUse(MaskReg)
+ .addUse(PassthruReg);
+ MIB.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
+bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &I) const {
+ assert(I.getNumExplicitDefs() == 0 && "Expected no defs for scatter");
+ // Operand indices (no explicit defs):
+ // 0: intrinsic ID
+ // 1: value vector
+ // 2: vector of pointers
+ // 3: alignment (i32 immediate)
+ // 4: mask (vector of i1)
+ Register ValuesReg = I.getOperand(1).getReg();
+ Register PtrsReg = I.getOperand(2).getReg();
+ uint32_t Alignment = I.getOperand(3).getImm();
+ Register MaskReg = I.getOperand(4).getReg();
+ Register AlignmentReg = buildI32Constant(Alignment, I);
+ MachineBasicBlock &BB = *I.getParent();
+
+ auto MIB =
+ BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedScatterINTEL))
+ .addUse(PtrsReg)
+ .addUse(AlignmentReg)
+ .addUse(MaskReg)
+ .addUse(ValuesReg);
+ MIB.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
+bool SPIRVInstructionSelector::diagnoseUnsupported(const MachineInstr &I,
+ const Twine &Msg) const {
+ const Function &F = I.getMF()->getFunction();
+ F.getContext().diagnose(
+ DiagnosticInfoUnsupported(F, Msg, I.getDebugLoc(), DS_Error));
+ return false;
+}
+
bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
SPIRVTypeInst ResType,
MachineInstr &I) const {
@@ -4355,6 +4425,16 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyFine);
case Intrinsic::spv_fwidth:
return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpFwidth);
+ case Intrinsic::spv_masked_gather:
+ if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
+ return selectMaskedGather(ResVReg, ResType, I);
+ return diagnoseUnsupported(
+ I, "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
+ case Intrinsic::spv_masked_scatter:
+ if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
+ return selectMaskedScatter(I);
+ return diagnoseUnsupported(
+ I, "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
default: {
std::string DiagMsg;
raw_string_ostream OS(DiagMsg);
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 93e82750c4f32..92b900be17642 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -356,6 +356,9 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
.legalFor({s1, s128})
.legalFor(allFloatAndIntScalarsAndPtrs)
.legalFor(allowedVectorTypes)
+ .legalIf([](const LegalityQuery &Query) {
+ return Query.Types[0].isPointerVector();
+ })
.moreElementsToNextPow2(0)
.fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),
LegalizeMutations::changeElementCountTo(
@@ -366,11 +369,25 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
getActionDefinitionsBuilder(G_INTTOPTR)
.legalForCartesianProduct(allPtrs, allIntScalars)
.legalIf(
- all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)));
+ all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)))
+ .legalIf([](const LegalityQuery &Query) {
+ const LLT DstTy = Query.Types[0];
+ const LLT SrcTy = Query.Types[1];
+ return DstTy.isPointerVector() && SrcTy.isVector() &&
+ !SrcTy.isPointer() &&
+ DstTy.getNumElements() == SrcTy.getNumElements();
+ });
getActionDefinitionsBuilder(G_PTRTOINT)
.legalForCartesianProduct(allIntScalars, allPtrs)
.legalIf(
- all(typeOfExtendedScalars(0, IsExtendedInts), typeInSet(1, allPtrs)));
+ all(typeOfExtendedScalars(0, IsExtendedInts), typeInSet(1, allPtrs)))
+ .legalIf([](const LegalityQuery &Query) {
+ const LLT DstTy = Query.Types[0];
+ const LLT SrcTy = Query.Types[1];
+ return SrcTy.isPointerVector() && DstTy.isVector() &&
+ !DstTy.isPointer() &&
+ DstTy.getNumElements() == SrcTy.getNumElements();
+ });
getActionDefinitionsBuilder(G_PTR_ADD)
.legalForCartesianProduct(allPtrs, allIntScalars)
.legalIf(
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 32446ab6909f6..b93a759ff4983 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -1496,6 +1496,15 @@ void addInstrRequirements(const MachineInstr &MI,
unsigned NumComponents = MI.getOperand(2).getImm();
if (NumComponents == 8 || NumComponents == 16)
Reqs.addCapability(SPIRV::Capability::Vector16);
+
+ assert(MI.getOperand(1).isReg());
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ SPIRVTypeInst ElemTypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
+ if (ElemTypeDef->getOpcode() == SPIRV::OpTypePointer &&
+ ST.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
+ Reqs.addCapability(SPIRV::Capability::MaskedGatherScatterINTEL);
+ }
break;
}
case SPIRV::OpTypePointer: {
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index e1a786ea16043..f1d115b424c97 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -397,6 +397,7 @@ defm SPV_NV_shader_atomic_fp16_vector
defm SPV_EXT_image_raw10_raw12 :ExtensionOperand<133, [EnvOpenCL, EnvVulkan]>;
defm SPV_ALTERA_arbitrary_precision_floating_point: ExtensionOperand<134, [EnvOpenCL]>;
defm SPV_KHR_fma : ExtensionOperand<135, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_masked_gather_scatter : ExtensionOperand<136, [EnvOpenCL]>;
//===----------------------------------------------------------------------===//
// Multiclass used to define Capabilities enum values and at the same time
@@ -620,6 +621,7 @@ defm PredicatedIOINTEL : CapabilityOperand<6257, 0, 0, [SPV_INTEL_predicated_io]
defm Int4TypeINTEL : CapabilityOperand<5112, 0, 0, [SPV_INTEL_int4], []>;
defm Int4CooperativeMatrixINTEL : CapabilityOperand<5114, 0, 0, [SPV_INTEL_int4], [Int4TypeINTEL, CooperativeMatrixKHR]>;
defm TensorFloat32RoundingINTEL : CapabilityOperand<6425, 0, 0, [SPV_INTEL_tensor_float32_conversion], []>;
+defm MaskedGatherScatterINTEL : CapabilityOperand<6427, 0, 0, [SPV_INTEL_masked_gather_scatter], []>;
defm BFloat16TypeKHR : CapabilityOperand<5116, 0, 0, [SPV_KHR_bfloat16], []>;
defm BFloat16DotProductKHR : CapabilityOperand<5117, 0, 0, [SPV_KHR_bfloat16], [BFloat16TypeKHR]>;
defm BFloat16CooperativeMatrixKHR : CapabilityOperand<5118, 0, 0, [SPV_KHR_bfloat16], [BFloat16TypeKHR, CooperativeMatrixKHR]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.cpp
index 95093d2b3c263..d69591377d315 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "SPIRVTargetTransformInfo.h"
+#include "SPIRVSubtarget.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
using namespace llvm;
@@ -38,3 +39,11 @@ Value *llvm::SPIRVTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
return nullptr;
}
}
+
+bool SPIRVTTIImpl::isLegalMaskedGather(Type *DataType, Align Alignment) const {
+ return ST->canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
+}
+
+bool SPIRVTTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) const {
+ return ST->canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h b/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
index 60c4e2de2fb23..35a1aa1922eed 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
@@ -61,6 +61,9 @@ class SPIRVTTIImpl final : public BasicTTIImplBase<SPIRVTTIImpl> {
Value *NewV) const override;
bool allowVectorElementIndexingUsingGEP() const override { return false; }
+
+ bool isLegalMaskedGather(Type *DataType, Align Alignment) const override;
+ bool isLegalMaskedScatter(Type *DataType, Align Alignment) const override;
};
} // namespace llvm
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-no-extension.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-no-extension.ll
new file mode 100644
index 0000000000000..f3e940f1a5ff2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-no-extension.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o /dev/null 2>&1 | FileCheck %s
+
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p1(<4 x ptr addrspace(1)>, i32, <4 x i1>, <4 x i32>)
+
+; CHECK: error: {{.*}}Vector of pointers requires SPV_INTEL_masked_gather_scatter extension
+
+define spir_kernel void @test_gather_no_ext(<4 x i64> %addrs) {
+entry:
+ %ptrs = inttoptr <4 x i64> %addrs to <4 x ptr addrspace(1)>
+ %data = call <4 x i32> @llvm.masked.gather.v4i32.v4p1(<4 x ptr addrspace(1)> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> zeroinitializer)
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-scatter.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-scatter.ll
new file mode 100644
index 0000000000000..add08059d0255
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-scatter.ll
@@ -0,0 +1,103 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - | FileCheck %s
+; TODO: spirv-val does not support vector operands in OpConvertPtrToU and OpConvertUToPtr with SPV_INTEL_masked_gather_scatter
+; RUNx: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpCapability MaskedGatherScatterINTEL
+; CHECK-DAG: OpExtension "SPV_INTEL_masked_gather_scatter"
+
+define spir_kernel void @test_gather_undef() {
+; CHECK-LABEL: Begin function test_gather_undef
+; CHECK: OpMaskedGatherINTEL
+entry:
+ %data = call <4 x i32> @llvm.masked.gather.v4i32.v4p1(<4 x ptr addrspace(1)> poison, i32 4, <4 x i1> poison, <4 x i32> poison)
+ ret void
+}
+
+define spir_kernel void @test_scatter_undef() {
+; CHECK-LABEL: Begin function test_scatter_undef
+; CHECK: OpMaskedScatterINTEL
+entry:
+ call void @llvm.masked.scatter.v4i32.v4p1(<4 x i32> poison, <4 x ptr addrspace(1)> poison, i32 4, <4 x i1> poison)
+ ret void
+}
+
+define spir_kernel void @test_gather_v4i32(<4 x i64> %addrs, <4 x i1> %mask, <4 x i32> %passthru) {
+; CHECK-LABEL: Begin function test_gather_v4i32
+; CHECK: OpMaskedGatherINTEL
+entry:
+ %ptrs = inttoptr <4 x i64> %addrs to <4 x ptr addrspace(1)>
+ %data = call <4 x i32> @llvm.masked.gather.v4i32.v4p1(<4 x ptr addrspace(1)> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %passthru)
+ ret void
+}
+
+define spir_kernel void @test_scatter_v4i32(<4 x i32> %data, <4 x i64> %addrs, <4 x i1> %mask) {
+; CHECK-LABEL: Begin function test_scatter_v4i32
+; CHECK: OpMaskedScatterINTEL
+entry:
+ %ptrs = inttoptr <4 x i64> %addrs to <4 x ptr addrspace(1)>
+ call void @llvm.masked.scatter.v4i32.v4p1(<4 x i32> %data, <4 x ptr addrspace(1)> %ptrs, i32 4, <4 x i1> %mask)
+ ret void
+}
+
+define spir_kernel void @test_gather_v2i64(<2 x i64> %addrs, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: Begin function test_gather_v2i64
+; CHECK: OpMaskedGatherINTEL
+entry:
+ %ptrs = inttoptr <2 x i64> %addrs to <2 x ptr addrspace(1)>
+ %data = call <2 x i64> @llvm.masked.gather.v2i64.v2p1(<2 x ptr addrspace(1)> %ptrs, i32 8, <2 x i1> %mask, <2 x i64> %passthru)
+ ret void
+}
+
+define spir_kernel void @test_scatter_v2i64(<2 x i64> %data, <2 x i64> %addrs, <2 x i1> %mask) {
+; CHECK-LABEL: Begin function test_scatter_v2i64
+; CHECK: OpMaskedScatterINTEL
+entry:
+ %ptrs = inttoptr <2 x i64> %addrs to <2 x ptr addrspace(1)>
+ call void @llvm.masked.scatter.v2i64.v2p1(<2 x i64> %data, <2 x ptr addrspace(1)> %ptrs, i32 8, <2 x i1> %mask)
+ ret void
+}
+
+define spir_kernel void @test_gather_v8i32(<8 x i64> %addrs, <8 x i1> %mask, <8 x i32> %passthru) {
+; CHECK-LABEL: Begin function test_gather_v8i32
+; CHECK: OpMaskedGatherINTEL
+entry:
+ %ptrs = inttoptr <8 x i64> %addrs to <8 x ptr addrspace(1)>
+ %data = call <8 x i32> @llvm.masked.gather.v8i32.v8p1(<8 x ptr addrspace(1)> %ptrs, i32 4, <8 x i1> %mask, <8 x i32> %passthru)
+ ret void
+}
+
+define spir_kernel void @test_scatter_v8i32(<8 x i32> %data, <8 x i64> %addrs, <8 x i1> %mask) {
+; CHECK-LABEL: Begin function test_scatter_v8i32
+; CHECK: OpMaskedScatterINTEL
+entry:
+ %ptrs = inttoptr <8 x i64> %addrs to <8 x ptr addrspace(1)>
+ call void @llvm.masked.scatter.v8i32.v8p1(<8 x i32> %data, <8 x ptr addrspace(1)> %ptrs, i32 4, <8 x i1> %mask)
+ ret void
+}
+
+define spir_kernel void @test_gather_v4f32(<4 x i64> %addrs, <4 x i1> %mask, <4 x float> %passthru) {
+; CHECK-LABEL: Begin function test_gather_v4f32
+; CHECK: OpMaskedGatherINTEL
+entry:
+ %ptrs = inttoptr <4 x i64> %addrs to <4 x ptr addrspace(1)>
+ %data = call <4 x float> @llvm.masked.gather.v4f32.v4p1(<4 x ptr addrspace(1)> %ptrs, i32 4, <4 x i1> %mask, <4 x float> %passthru)
+ ret void
+}
+
+define spir_kernel void @test_scatter_v4f32(<4 x float> %data, <4 x i64> %addrs, <4 x i1> %mask) {
+; CHECK-LABEL: Begin function test_scatter_v4f32
+; CHECK: OpMaskedScatterINTEL
+entry:
+ %ptrs = inttoptr <4 x i64> %addrs to <4 x ptr addrspace(1)>
+ call void @llvm.masked.scatter.v4f32.v4p1(<4 x float> %data, <4 x ptr addrspace(1)> %ptrs, i32 4, <4 x i1> %mask)
+ ret void
+}
+
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p1(<4 x ptr addrspace(1)>, i32, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.scatter.v4i32.v4p1(<4 x i32>, <4 x ptr addrspace(1)>, i32, <4 x i1>)
+declare <2 x i64> @llvm.masked.gather.v2i64.v2p1(<2 x ptr addrspace(1)>, i32, <2 x i1>, <2 x i64>)
+declare void @llvm.masked.scatter.v2i64.v2p1(<2 x i64>, <2 x ptr addrspace(1)>, i32, <2 x i1>)
+declare <8 x i32> @llvm.masked.gather.v8i32.v8p1(<8 x ptr addrspace(1)>, i32, <8 x i1>, <8 x i32>)
+declare void @llvm.masked.scatter.v8i32.v8p1(<8 x i32>, <8 x ptr addrspace(1)>, i32, <8 x i1>)
+declare <4 x float> @llvm.masked.gather.v4f32.v4p1(<4 x ptr addrspace(1)>, i32, <4 x i1>, <4 x float>)
+declare void @llvm.masked.scatter.v4f32.v4p1(<4 x float>, <4 x ptr addrspace(1)>, i32, <4 x i1>)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-scatter-no-extension.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-scatter-no-extension.ll
new file mode 100644
index 0000000000000..4cbef7f905047
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-scatter-no-extension.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @llvm.masked.scatter.v4i32.v4p1(<4 x i32>, <4 x ptr addrspace(1)>, i32, <4 x i1>)
+
+; CHECK: error: {{.*}}Vector of pointers requires SPV_INTEL_masked_gather_scatter extension
+
+define spir_kernel void @test_scatter_no_ext(<4 x i32> %data, <4 x i64> %addrs) {
+entry:
+ %ptrs = inttoptr <4 x i64> %addrs to <4 x ptr addrspace(1)>
+ call void @llvm.masked.scatter.v4i32.v4p1(<4 x i32> %data, <4 x ptr addrspace(1)> %ptrs, i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 false>)
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-no-extension.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-no-extension.ll
new file mode 100644
index 0000000000000..d892b3487b725
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-no-extension.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: error:{{.*}}Vector of pointers requires SPV_INTEL_masked_gather_scatter extension
+
+declare spir_func void @foo(<2 x i64>)
+
+define spir_kernel void @test_ptrtoint(<2 x ptr addrspace(1)> %p) {
+entry:
+ %addr = ptrtoint <2 x ptr addrspace(1)> %p to <2 x i64>
+ call void @foo(<2 x i64> %addr)
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-ptrtoint.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-ptrtoint.ll
new file mode 100644
index 0000000000000..74988e07b537b
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-ptrtoint.ll
@@ -0,0 +1,33 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - | FileCheck %s
+; TODO: spirv-val does not support vector operands in OpConvertPtrToU and OpConvertUToPtr with SPV_INTEL_masked_gather_scatter
+; RUNx: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpCapability MaskedGatherScatterINTEL
+; CHECK: OpExtension "SPV_INTEL_masked_gather_scatter"
+
+; CHECK-DAG: %[[#Int64:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#PtrTy:]] = OpTypePointer CrossWorkgroup
+; CHECK-DAG: %[[#Vec2Ptr:]] = OpTypeVector %[[#PtrTy]] 2
+; CHECK-DAG: %[[#Vec2Int64:]] = OpTypeVector %[[#Int64]] 2
+
+declare spir_func void @foo(<2 x i64>)
+
+define spir_kernel void @test_ptrtoint_vec2(<2 x ptr addrspace(1)> %p) {
+; CHECK-LABEL: Begin function test_ptrtoint_vec2
+; CHECK: OpConvertPtrToU %[[#Vec2Int64]]
+entry:
+ %addr = ptrtoint <2 x ptr addrspace(1)> %p to <2 x i64>
+ call void @foo(<2 x i64> %addr)
+ ret void
+}
+
+declare spir_func void @bar(<2 x ptr addrspace(1)>)
+
+define spir_kernel void @test_inttoptr_vec2(<2 x i64> %addr) {
+; CHECK-LABEL: Begin function test_inttoptr_vec2
+; CHECK: OpConvertUToPtr %[[#Vec2Ptr]]
+entry:
+ %p = inttoptr <2 x i64> %addr to <2 x ptr addrspace(1)>
+ call void @bar(<2 x ptr addrspace(1)> %p)
+ ret void
+}
More information about the llvm-commits
mailing list