[llvm] [SPIR-V] Add support for SPV_INTEL_masked_gather_scatter extension (PR #185418)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 9 06:36:15 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-ir
Author: Arseniy Obolenskiy (aobolensk)
<details>
<summary>Changes</summary>
Fixes the first bullet in #<!-- -->184638
Corresponding patch to add support for vector operands in OpConvertPtrToU/OpConvertUToPtr operations in spirv-val: https://github.com/KhronosGroup/SPIRV-Tools/pull/6575
SPIR-V extension reference used: https://github.com/KhronosGroup/SPIRV-Registry/blob/278044a51fee280bfc91322cdb55b51357db5cb8/extensions/INTEL/SPV_INTEL_masked_gather_scatter.asciidoc
---
Patch is 31.20 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/185418.diff
17 Files Affected:
- (modified) llvm/include/llvm/IR/IntrinsicsSPIRV.td (+8)
- (modified) llvm/lib/Target/SPIRV/CMakeLists.txt (+1)
- (modified) llvm/lib/Target/SPIRV/SPIRV.h (+3)
- (modified) llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp (+2)
- (added) llvm/lib/Target/SPIRV/SPIRVConvertMaskedMemIntrinsics.cpp (+136)
- (modified) llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp (+19-10)
- (modified) llvm/lib/Target/SPIRV/SPIRVInstrInfo.td (+6)
- (modified) llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp (+68)
- (modified) llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp (+19-2)
- (modified) llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp (+21)
- (modified) llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td (+2)
- (modified) llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp (+2)
- (added) llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-scatter-no-extension.ll (+12)
- (added) llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/masked-gather-scatter.ll (+103)
- (added) llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-no-extension.ll (+13)
- (added) llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/vector-of-pointers-ptrtoint.ll (+33)
- (modified) llvm/test/CodeGen/SPIRV/llc-pipeline.ll (+2)
``````````diff
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 3fc18a254f672..df3dceede04b1 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -44,6 +44,14 @@ let TargetPrefix = "spv" in {
def int_spv_undef : Intrinsic<[llvm_i32_ty], []>;
def int_spv_inline_asm : Intrinsic<[], [llvm_metadata_ty, llvm_metadata_ty, llvm_vararg_ty]>;
+ // Masked Gather/Scatter (SPV_INTEL_masked_gather_scatter)
+ def int_spv_masked_gather : Intrinsic<[llvm_any_ty],
+ [llvm_any_ty, llvm_i32_ty, llvm_any_ty, llvm_any_ty],
+ [IntrReadMem, IntrWillReturn, ImmArg<ArgIndex<1>>]>;
+ def int_spv_masked_scatter : Intrinsic<[],
+ [llvm_any_ty, llvm_any_ty, llvm_i32_ty, llvm_any_ty],
+ [IntrWriteMem, IntrWillReturn, ImmArg<ArgIndex<2>>]>;
+
// Expect, Assume Intrinsics
def int_spv_assume : Intrinsic<[], [llvm_i1_ty]>;
def int_spv_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>;
diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
index 58989237ad3ea..f492f4bb6507e 100644
--- a/llvm/lib/Target/SPIRV/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -22,6 +22,7 @@ add_llvm_target(SPIRVCodeGen
SPIRVCallLowering.cpp
SPIRVInlineAsmLowering.cpp
SPIRVCommandLine.cpp
+ SPIRVConvertMaskedMemIntrinsics.cpp
SPIRVEmitIntrinsics.cpp
SPIRVGlobalRegistry.cpp
SPIRVInstrInfo.cpp
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index da4da1a3fe83b..b36d4f9cab31c 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -32,6 +32,8 @@ FunctionPass *createSPIRVRegularizerPass();
FunctionPass *createSPIRVPreLegalizerCombiner();
FunctionPass *createSPIRVPreLegalizerPass();
FunctionPass *createSPIRVPostLegalizerPass();
+FunctionPass *
+createSPIRVConvertMaskedMemIntrinsicsPass(const SPIRVTargetMachine *TM);
ModulePass *createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM);
ModulePass *createSPIRVPrepareGlobalsPass();
MachineFunctionPass *createSPIRVEmitNonSemanticDIPass(SPIRVTargetMachine *TM);
@@ -59,6 +61,7 @@ void initializeSPIRVPrepareGlobalsPass(PassRegistry &);
void initializeSPIRVStripConvergentIntrinsicsPass(PassRegistry &);
void initializeSPIRVLegalizeImplicitBindingPass(PassRegistry &);
void initializeSPIRVLegalizeZeroSizeArraysLegacyPass(PassRegistry &);
+void initializeSPIRVConvertMaskedMemIntrinsicsPass(PassRegistry &);
} // namespace llvm
#endif // LLVM_LIB_TARGET_SPIRV_SPIRV_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index 33e1b52b724e6..734a03ff60141 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -86,6 +86,8 @@ static const StringMap<SPIRV::Extension::Extension> SPIRVExtensionMap = {
SPIRV::Extension::Extension::SPV_INTEL_memory_access_aliasing},
{"SPV_INTEL_joint_matrix",
SPIRV::Extension::Extension::SPV_INTEL_joint_matrix},
+ {"SPV_INTEL_masked_gather_scatter",
+ SPIRV::Extension::Extension::SPV_INTEL_masked_gather_scatter},
{"SPV_KHR_16bit_storage",
SPIRV::Extension::Extension::SPV_KHR_16bit_storage},
{"SPV_KHR_device_group", SPIRV::Extension::Extension::SPV_KHR_device_group},
diff --git a/llvm/lib/Target/SPIRV/SPIRVConvertMaskedMemIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVConvertMaskedMemIntrinsics.cpp
new file mode 100644
index 0000000000000..aa1a83657bff8
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVConvertMaskedMemIntrinsics.cpp
@@ -0,0 +1,136 @@
+//===- SPIRVConvertMaskedMemIntrinsics.cpp - Convert masked mem intrinsics ==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass converts llvm.masked.gather/scatter to spv.masked.gather/scatter
+// to prevent them from being scalarized by the generic scalarization pass.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SPIRV.h"
+#include "SPIRVSubtarget.h"
+#include "SPIRVTargetMachine.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsSPIRV.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "spirv-convert-masked-mem-intrinsics"
+
+namespace {
+
+class SPIRVConvertMaskedMemIntrinsics
+ : public FunctionPass,
+ public InstVisitor<SPIRVConvertMaskedMemIntrinsics> {
+ const SPIRVTargetMachine *TM = nullptr;
+
+public:
+ static char ID;
+
+ SPIRVConvertMaskedMemIntrinsics() : FunctionPass(ID) {
+ initializeSPIRVConvertMaskedMemIntrinsicsPass(
+ *PassRegistry::getPassRegistry());
+ }
+
+ SPIRVConvertMaskedMemIntrinsics(const SPIRVTargetMachine *TM)
+ : FunctionPass(ID), TM(TM) {
+ initializeSPIRVConvertMaskedMemIntrinsicsPass(
+ *PassRegistry::getPassRegistry());
+ }
+
+ bool runOnFunction(Function &F) override;
+ void visitIntrinsicInst(IntrinsicInst &I);
+
+ StringRef getPassName() const override {
+ return "SPIRV convert masked memory intrinsics";
+ }
+
+private:
+ SmallVector<Instruction *, 4> ToErase;
+};
+
+} // namespace
+
+char SPIRVConvertMaskedMemIntrinsics::ID = 0;
+
+INITIALIZE_PASS(SPIRVConvertMaskedMemIntrinsics,
+ "spirv-convert-masked-mem-intrinsics",
+ "Convert masked memory intrinsics for SPIR-V", false, false)
+
+bool SPIRVConvertMaskedMemIntrinsics::runOnFunction(Function &F) {
+ if (!TM)
+ return false;
+
+ ToErase.clear();
+ visit(F);
+
+ for (Instruction *I : ToErase)
+ I->eraseFromParent();
+
+ return !ToErase.empty();
+}
+
+void SPIRVConvertMaskedMemIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
+ if (I.getIntrinsicID() == Intrinsic::masked_gather) {
+ const SPIRVSubtarget &ST =
+ TM->getSubtarget<SPIRVSubtarget>(*I.getFunction());
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
+ report_fatal_error(
+ "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
+
+ IRBuilder<> B(I.getParent());
+ B.SetInsertPoint(&I);
+
+ Value *Ptrs = I.getArgOperand(0);
+ Value *Mask = I.getArgOperand(1);
+ Value *Passthru = I.getArgOperand(2);
+
+ // Alignment is stored as a parameter attribute, not as a regular parameter
+ uint32_t Alignment = I.getParamAlign(0).valueOrOne().value();
+
+ SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(Alignment), Mask,
+ Passthru};
+ SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
+ Mask->getType(), Passthru->getType()};
+
+ auto *NewI = B.CreateIntrinsic(Intrinsic::spv_masked_gather, Types, Args);
+ I.replaceAllUsesWith(NewI);
+ ToErase.push_back(&I);
+ } else if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
+ const SPIRVSubtarget &ST =
+ TM->getSubtarget<SPIRVSubtarget>(*I.getFunction());
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
+ report_fatal_error(
+ "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
+
+ IRBuilder<> B(I.getParent());
+ B.SetInsertPoint(&I);
+
+ Value *Values = I.getArgOperand(0);
+ Value *Ptrs = I.getArgOperand(1);
+ Value *Mask = I.getArgOperand(2);
+
+ // Alignment is stored as a parameter attribute on the ptrs parameter (arg
+ // 1)
+ uint32_t Alignment = I.getParamAlign(1).valueOrOne().value();
+
+ SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(Alignment), Mask};
+ SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
+ Mask->getType()};
+
+ B.CreateIntrinsic(Intrinsic::spv_masked_scatter, Types, Args);
+ ToErase.push_back(&I);
+ }
+}
+
+FunctionPass *
+llvm::createSPIRVConvertMaskedMemIntrinsicsPass(const SPIRVTargetMachine *TM) {
+ return new SPIRVConvertMaskedMemIntrinsics(TM);
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 9a85634c82626..c60663b43bdc4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -318,17 +318,26 @@ SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVTypeInst ElemType,
auto EleOpc = ElemType->getOpcode();
(void)EleOpc;
assert(NumElems >= 2 && "SPIR-V OpTypeVector requires at least 2 components");
- assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||
- EleOpc == SPIRV::OpTypeBool) &&
- "Invalid vector element type");
- return createConstOrTypeAtFunctionEntry(MIRBuilder, [&](MachineIRBuilder
- &MIRBuilder) {
- return MIRBuilder.buildInstr(SPIRV::OpTypeVector)
- .addDef(createTypeVReg(MIRBuilder))
- .addUse(getSPIRVTypeID(ElemType))
- .addImm(NumElems);
- });
+ if (EleOpc == SPIRV::OpTypePointer) {
+ assert(cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget())
+ .canUseExtension(
+ SPIRV::Extension::SPV_INTEL_masked_gather_scatter) &&
+ "Vector of pointers requires SPV_INTEL_masked_gather_scatter "
+ "extension");
+ } else {
+ assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||
+ EleOpc == SPIRV::OpTypeBool) &&
+ "Invalid vector element type");
+ }
+
+ return createConstOrTypeAtFunctionEntry(
+ MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+ return MIRBuilder.buildInstr(SPIRV::OpTypeVector)
+ .addDef(createTypeVReg(MIRBuilder))
+ .addUse(getSPIRVTypeID(ElemType))
+ .addImm(NumElems);
+ });
}
Register SPIRVGlobalRegistry::getOrCreateConstFP(APFloat Val, MachineInstr &I,
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index d2f81bc30e949..819cdd6107d0d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -1131,3 +1131,9 @@ def OpFixedLogALTERA: Op<5932, (outs ID:$res), (ins TYPE:$result_type, ID:$input
"$res = OpFixedLogALTERA $result_type $input $sign $l $rl $q $o">;
def OpFixedExpALTERA: Op<5933, (outs ID:$res), (ins TYPE:$result_type, ID:$input, i32imm:$sign, i32imm:$l, i32imm:$rl, i32imm:$q, i32imm:$o),
"$res = OpFixedExpALTERA $result_type $input $sign $l $rl $q $o">;
+
+//SPV_INTEL_masked_gather_scatter
+def OpMaskedGatherINTEL: Op<6428, (outs ID:$res), (ins TYPE:$resType, ID:$ptrs, ID:$alignment, ID:$mask, ID:$fillEmpty),
+ "$res = OpMaskedGatherINTEL $resType $ptrs $alignment $mask $fillEmpty">;
+def OpMaskedScatterINTEL: Op<6429, (outs), (ins ID:$ptrs, ID:$alignment, ID:$mask, ID:$values),
+ "OpMaskedScatterINTEL $ptrs $alignment $mask $values">;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 7b4c047593a3a..7896b1877362c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -300,6 +300,10 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectGEP(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
+ bool selectMaskedGather(Register ResVReg, SPIRVTypeInst ResType,
+ MachineInstr &I) const;
+ bool selectMaskedScatter(MachineInstr &I) const;
+
bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType,
@@ -1722,6 +1726,60 @@ bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
return true;
}
+bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg,
+ SPIRVTypeInst ResType,
+ MachineInstr &I) const {
+ assert(I.getNumExplicitDefs() == 1 && "Expected single def for gather");
+ // Operand indices (after explicit defs):
+ // 0: intrinsic ID
+ // 1: vector of pointers
+ // 2: alignment (i32 immediate)
+ // 3: mask (vector of i1)
+ // 4: passthru/fill value
+ Register PtrsReg = I.getOperand(I.getNumExplicitDefs() + 1).getReg();
+ uint32_t Alignment = I.getOperand(I.getNumExplicitDefs() + 2).getImm();
+ Register MaskReg = I.getOperand(I.getNumExplicitDefs() + 3).getReg();
+ Register PassthruReg = I.getOperand(I.getNumExplicitDefs() + 4).getReg();
+ Register AlignmentReg = buildI32Constant(Alignment, I);
+
+ MachineBasicBlock &BB = *I.getParent();
+ auto MIB =
+ BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedGatherINTEL))
+ .addDef(ResVReg)
+ .addUse(GR.getSPIRVTypeID(ResType))
+ .addUse(PtrsReg)
+ .addUse(AlignmentReg)
+ .addUse(MaskReg)
+ .addUse(PassthruReg);
+ MIB.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
+bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &I) const {
+ assert(I.getNumExplicitDefs() == 0 && "Expected no defs for scatter");
+ // Operand indices (no explicit defs):
+ // 0: intrinsic ID
+ // 1: value vector
+ // 2: vector of pointers
+ // 3: alignment (i32 immediate)
+ // 4: mask (vector of i1)
+ Register ValuesReg = I.getOperand(1).getReg();
+ Register PtrsReg = I.getOperand(2).getReg();
+ uint32_t Alignment = I.getOperand(3).getImm();
+ Register MaskReg = I.getOperand(4).getReg();
+ Register AlignmentReg = buildI32Constant(Alignment, I);
+ MachineBasicBlock &BB = *I.getParent();
+
+ auto MIB =
+ BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedScatterINTEL))
+ .addUse(PtrsReg)
+ .addUse(AlignmentReg)
+ .addUse(MaskReg)
+ .addUse(ValuesReg);
+ MIB.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
SPIRVTypeInst ResType,
MachineInstr &I) const {
@@ -4319,6 +4377,16 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyFine);
case Intrinsic::spv_fwidth:
return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpFwidth);
+ case Intrinsic::spv_masked_gather:
+ if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
+ return selectMaskedGather(ResVReg, ResType, I);
+ report_fatal_error(
+ "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
+ case Intrinsic::spv_masked_scatter:
+ if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
+ return selectMaskedScatter(I);
+ report_fatal_error(
+ "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
default: {
std::string DiagMsg;
raw_string_ostream OS(DiagMsg);
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 93e82750c4f32..92b900be17642 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -356,6 +356,9 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
.legalFor({s1, s128})
.legalFor(allFloatAndIntScalarsAndPtrs)
.legalFor(allowedVectorTypes)
+ .legalIf([](const LegalityQuery &Query) {
+ return Query.Types[0].isPointerVector();
+ })
.moreElementsToNextPow2(0)
.fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),
LegalizeMutations::changeElementCountTo(
@@ -366,11 +369,25 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
getActionDefinitionsBuilder(G_INTTOPTR)
.legalForCartesianProduct(allPtrs, allIntScalars)
.legalIf(
- all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)));
+ all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)))
+ .legalIf([](const LegalityQuery &Query) {
+ const LLT DstTy = Query.Types[0];
+ const LLT SrcTy = Query.Types[1];
+ return DstTy.isPointerVector() && SrcTy.isVector() &&
+ !SrcTy.isPointer() &&
+ DstTy.getNumElements() == SrcTy.getNumElements();
+ });
getActionDefinitionsBuilder(G_PTRTOINT)
.legalForCartesianProduct(allIntScalars, allPtrs)
.legalIf(
- all(typeOfExtendedScalars(0, IsExtendedInts), typeInSet(1, allPtrs)));
+ all(typeOfExtendedScalars(0, IsExtendedInts), typeInSet(1, allPtrs)))
+ .legalIf([](const LegalityQuery &Query) {
+ const LLT DstTy = Query.Types[0];
+ const LLT SrcTy = Query.Types[1];
+ return SrcTy.isPointerVector() && DstTy.isVector() &&
+ !DstTy.isPointer() &&
+ DstTy.getNumElements() == SrcTy.getNumElements();
+ });
getActionDefinitionsBuilder(G_PTR_ADD)
.legalForCartesianProduct(allPtrs, allIntScalars)
.legalIf(
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 923f59917cb10..0d89a3e7e16be 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -1496,6 +1496,15 @@ void addInstrRequirements(const MachineInstr &MI,
unsigned NumComponents = MI.getOperand(2).getImm();
if (NumComponents == 8 || NumComponents == 16)
Reqs.addCapability(SPIRV::Capability::Vector16);
+
+ assert(MI.getOperand(1).isReg());
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ SPIRVTypeInst ElemTypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
+ if (ElemTypeDef->getOpcode() == SPIRV::OpTypePointer &&
+ ST.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
+ Reqs.addCapability(SPIRV::Capability::MaskedGatherScatterINTEL);
+ }
break;
}
case SPIRV::OpTypePointer: {
@@ -1864,6 +1873,18 @@ void addInstrRequirements(const MachineInstr &MI,
case SPIRV::OpAtomicFMaxEXT:
AddAtomicFloatRequirements(MI, Reqs, ST);
break;
+ case SPIRV::OpConvertPtrToU:
+ case SPIRV::OpConvertUToPtr: {
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ SPIRVTypeInst ResultType = MRI.getVRegDef(MI.getOperand(1).getReg());
+ if (ResultType->getOpcode() == SPIRV::OpTypeVector &&
+ ST.canUseExtension(
+ SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
+ Reqs.addCapability(SPIRV::Capability::MaskedGatherScatterINTEL);
+ }
+ break;
+ }
case SPIRV::OpConvertBF16ToFINTEL:
case SPIRV::OpConvertFToBF16INTEL:
if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index e1a786ea16043..f1d115b424c97 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -397,6 +397,7 @@ defm SPV_NV_shader_atomic_fp16_vector
defm SPV_EXT_image_raw10_raw12 :ExtensionOperand<133, [EnvOpenCL, EnvVulkan]>;
defm SPV_ALTERA_arbitrary_precision_floating_point: ExtensionOperand<134, [EnvOpenCL]>;
defm SPV_KHR_fma : ExtensionOperand<135, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_masked_gather_scatter : ExtensionOperand<136, [EnvOpenCL]>;
//===----------------------------------------------------------------------===//
// Multiclass used to define Capabilities enum values and at the same time
@@ -620,6 +621,7 @@ defm PredicatedIOINTEL : CapabilityOperand<6257, 0, 0, [SPV_INTEL_predicated_io]
defm Int4TypeINTEL : CapabilityOperand<5112, 0, 0, [SPV_INTEL_int4], []>;
defm Int4CooperativeMatrixINTEL : CapabilityOperand<5114, 0, 0, [SPV_INTEL_int4], [Int4TypeINTEL, CooperativeMatrixKHR]>;
defm Ten...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/185418
More information about the llvm-commits
mailing list