[llvm] d21a35a - [VP] Implementation of intrinsic and SDNode definitions for VP load, store, gather, scatter.
Simon Moll via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 1 04:35:34 PDT 2021
Author: Hussain Kadhem
Date: 2021-07-01T13:34:44+02:00
New Revision: d21a35ac0a958fd4cff0b8f424a2706b8785b89d
URL: https://github.com/llvm/llvm-project/commit/d21a35ac0a958fd4cff0b8f424a2706b8785b89d
DIFF: https://github.com/llvm/llvm-project/commit/d21a35ac0a958fd4cff0b8f424a2706b8785b89d.diff
LOG: [VP] Implementation of intrinsic and SDNode definitions for VP load, store, gather, scatter.
This patch adds intrinsic definitions and SDNodes for predicated
load/store/gather/scatter, based on the work done in D57504.
Reviewed By: simoll, craig.topper
Differential Revision: https://reviews.llvm.org/D99355
Added:
Modified:
llvm/include/llvm/CodeGen/SelectionDAGNodes.h
llvm/include/llvm/IR/IntrinsicInst.h
llvm/include/llvm/IR/Intrinsics.td
llvm/include/llvm/IR/VPIntrinsics.def
llvm/lib/IR/IntrinsicInst.cpp
llvm/unittests/IR/VPIntrinsicTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index dba39a361a68b..929bcb4bd5f10 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1367,33 +1367,36 @@ class MemSDNode : public SDNode {
static bool classof(const SDNode *N) {
// For some targets, we lower some target intrinsics to a MemIntrinsicNode
// with either an intrinsic or a target opcode.
- return N->getOpcode() == ISD::LOAD ||
- N->getOpcode() == ISD::STORE ||
- N->getOpcode() == ISD::PREFETCH ||
- N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
- N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
- N->getOpcode() == ISD::ATOMIC_SWAP ||
- N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
- N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
- N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
- N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
- N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
- N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
- N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
- N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
- N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
- N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
- N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
- N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
- N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
- N->getOpcode() == ISD::ATOMIC_LOAD ||
- N->getOpcode() == ISD::ATOMIC_STORE ||
- N->getOpcode() == ISD::MLOAD ||
- N->getOpcode() == ISD::MSTORE ||
- N->getOpcode() == ISD::MGATHER ||
- N->getOpcode() == ISD::MSCATTER ||
- N->isMemIntrinsic() ||
- N->isTargetMemoryOpcode();
+ switch (N->getOpcode()) {
+ case ISD::LOAD:
+ case ISD::STORE:
+ case ISD::PREFETCH:
+ case ISD::ATOMIC_CMP_SWAP:
+ case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
+ case ISD::ATOMIC_SWAP:
+ case ISD::ATOMIC_LOAD_ADD:
+ case ISD::ATOMIC_LOAD_SUB:
+ case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_CLR:
+ case ISD::ATOMIC_LOAD_OR:
+ case ISD::ATOMIC_LOAD_XOR:
+ case ISD::ATOMIC_LOAD_NAND:
+ case ISD::ATOMIC_LOAD_MIN:
+ case ISD::ATOMIC_LOAD_MAX:
+ case ISD::ATOMIC_LOAD_UMIN:
+ case ISD::ATOMIC_LOAD_UMAX:
+ case ISD::ATOMIC_LOAD_FADD:
+ case ISD::ATOMIC_LOAD_FSUB:
+ case ISD::ATOMIC_LOAD:
+ case ISD::ATOMIC_STORE:
+ case ISD::MLOAD:
+ case ISD::MSTORE:
+ case ISD::MGATHER:
+ case ISD::MSCATTER:
+ return true;
+ default:
+ return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
+ }
}
};
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 606c2b570c1b7..2dbbda2aa7dfd 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -403,21 +403,34 @@ class VPIntrinsic : public IntrinsicInst {
// Whether \p ID is a VP intrinsic ID.
static bool isVPIntrinsic(Intrinsic::ID);
- /// \return the mask parameter or nullptr.
+ /// \return The mask parameter or nullptr.
Value *getMaskParam() const;
void setMaskParam(Value *);
- /// \return the vector length parameter or nullptr.
+ /// \return The vector length parameter or nullptr.
Value *getVectorLengthParam() const;
void setVectorLengthParam(Value *);
- /// \return whether the vector length param can be ignored.
+ /// \return Whether the vector length param can be ignored.
bool canIgnoreVectorLengthParam() const;
- /// \return the static element count (vector number of elements) the vector
+ /// \return The static element count (vector number of elements) the vector
/// length parameter applies to.
ElementCount getStaticVectorLength() const;
+ /// \return The alignment of the pointer used by this load/store/gather or
+ /// scatter.
+ MaybeAlign getPointerAlignment() const;
+ // MaybeAlign setPointerAlignment(Align NewAlign); // TODO
+
+ /// \return The pointer operand of this load,store, gather or scatter.
+ Value *getMemoryPointerParam() const;
+ static Optional<unsigned> getMemoryPointerParamPos(Intrinsic::ID);
+
+ /// \return The data (payload) operand of this store or scatter.
+ Value *getMemoryDataParam() const;
+ static Optional<unsigned> getMemoryDataParamPos(Intrinsic::ID);
+
// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const IntrinsicInst *I) {
return isVPIntrinsic(I->getIntrinsicID());
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index c7bdd86d82f83..975a109526353 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1368,6 +1368,32 @@ def int_experimental_stepvector : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
[], [IntrNoMem]>;
//===---------------- Vector Predication Intrinsics --------------===//
+// Memory Intrinsics
+def int_vp_store : DefaultAttrsIntrinsic<[],
+ [ llvm_anyvector_ty,
+ LLVMAnyPointerType<LLVMMatchType<0>>,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty],
+ [ NoCapture<ArgIndex<1>>, IntrNoSync, IntrWriteMem, IntrArgMemOnly, IntrWillReturn ]>;
+
+def int_vp_load : DefaultAttrsIntrinsic<[ llvm_anyvector_ty],
+ [ LLVMAnyPointerType<LLVMMatchType<0>>,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty],
+ [ NoCapture<ArgIndex<0>>, IntrNoSync, IntrReadMem, IntrWillReturn, IntrArgMemOnly ]>;
+
+def int_vp_gather: DefaultAttrsIntrinsic<[ llvm_anyvector_ty],
+ [ LLVMVectorOfAnyPointersToElt<0>,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty],
+ [ IntrReadMem, IntrNoSync, IntrWillReturn, IntrArgMemOnly ]>;
+
+def int_vp_scatter: DefaultAttrsIntrinsic<[],
+ [ llvm_anyvector_ty,
+ LLVMVectorOfAnyPointersToElt<0>,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty],
+ [ IntrArgMemOnly, IntrNoSync, IntrWillReturn ]>; // TODO allow IntrNoCapture for vectors of pointers
// Speculatable Binary operators
let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in {
diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def
index f6a283c800eb9..92e2cd3a27830 100644
--- a/llvm/include/llvm/IR/VPIntrinsics.def
+++ b/llvm/include/llvm/IR/VPIntrinsics.def
@@ -100,6 +100,17 @@ END_REGISTER_VP_SDNODE(SDOPC)
#define HANDLE_VP_TO_CONSTRAINEDFP(HASROUND, HASEXCEPT, INTRINID)
#endif
+// Map this VP intrinsic to its canonical functional intrinsic.
+#ifndef HANDLE_VP_TO_INTRIN
+#define HANDLE_VP_TO_INTRIN(ID)
+#endif
+
+// This VP Intrinsic is a memory operation
+// The pointer arg is at POINTERPOS and the data arg is at DATAPOS.
+#ifndef HANDLE_VP_IS_MEMOP
+#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS)
+#endif
+
/// } Property Macros
///// Integer Arithmetic {
@@ -191,6 +202,36 @@ HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem)
///// } Floating-Point Arithmetic
+///// Memory Operations {
+// llvm.vp.store(ptr,val,mask,vlen)
+BEGIN_REGISTER_VP(vp_store, 2, 3, VP_STORE, 0)
+HANDLE_VP_TO_OPC(Store)
+HANDLE_VP_TO_INTRIN(masked_store)
+HANDLE_VP_IS_MEMOP(vp_store, 1, 0)
+END_REGISTER_VP(vp_store, VP_STORE)
+
+// llvm.vp.scatter(ptr,val,mask,vlen)
+BEGIN_REGISTER_VP(vp_scatter, 2, 3, VP_SCATTER, 0)
+HANDLE_VP_TO_INTRIN(masked_scatter)
+HANDLE_VP_IS_MEMOP(vp_scatter, 1, 0)
+END_REGISTER_VP(vp_scatter, VP_SCATTER)
+
+// llvm.vp.load(ptr,mask,vlen)
+BEGIN_REGISTER_VP(vp_load, 1, 2, VP_LOAD, -1)
+HANDLE_VP_TO_OPC(Load)
+HANDLE_VP_TO_INTRIN(masked_load)
+HANDLE_VP_IS_MEMOP(vp_load, 0, None)
+END_REGISTER_VP(vp_load, VP_LOAD)
+
+// llvm.vp.gather(ptr,mask,vlen)
+BEGIN_REGISTER_VP(vp_gather, 1, 2, VP_GATHER, -1)
+HANDLE_VP_TO_INTRIN(masked_gather)
+HANDLE_VP_IS_MEMOP(vp_gather, 0, None)
+END_REGISTER_VP(vp_gather, VP_GATHER)
+
+///// } Memory Operations
+
+
#undef BEGIN_REGISTER_VP
#undef BEGIN_REGISTER_VP_INTRINSIC
#undef BEGIN_REGISTER_VP_SDNODE
@@ -199,3 +240,5 @@ HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem)
#undef END_REGISTER_VP_SDNODE
#undef HANDLE_VP_TO_OPC
#undef HANDLE_VP_TO_CONSTRAINEDFP
+#undef HANDLE_VP_TO_INTRIN
+#undef HANDLE_VP_IS_MEMOP
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 776590af9a32e..19942fa187fd2 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -1,4 +1,4 @@
-//===-- InstrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
+//===-- IntrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -340,6 +340,53 @@ VPIntrinsic::getVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
}
}
+/// \return the alignment of the pointer used by this load/store/gather or
+/// scatter.
+MaybeAlign VPIntrinsic::getPointerAlignment() const {
+ Optional<unsigned> PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID());
+ assert(PtrParamOpt.hasValue() && "no pointer argument!");
+ return getParamAlign(PtrParamOpt.getValue());
+}
+
+/// \return The pointer operand of this load,store, gather or scatter.
+Value *VPIntrinsic::getMemoryPointerParam() const {
+ if (auto PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID()))
+ return getArgOperand(PtrParamOpt.getValue());
+ return nullptr;
+}
+
+Optional<unsigned> VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) {
+ switch (VPID) {
+ default:
+ return None;
+
+#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \
+ case Intrinsic::VPID: \
+ return POINTERPOS;
+#include "llvm/IR/VPIntrinsics.def"
+ }
+}
+
+/// \return The data (payload) operand of this store or scatter.
+Value *VPIntrinsic::getMemoryDataParam() const {
+ auto DataParamOpt = getMemoryDataParamPos(getIntrinsicID());
+ if (!DataParamOpt.hasValue())
+ return nullptr;
+ return getArgOperand(DataParamOpt.getValue());
+}
+
+Optional<unsigned> VPIntrinsic::getMemoryDataParamPos(Intrinsic::ID VPID) {
+ switch (VPID) {
+ default:
+ return None;
+
+#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \
+ case Intrinsic::VPID: \
+ return DATAPOS;
+#include "llvm/IR/VPIntrinsics.def"
+ }
+}
+
bool VPIntrinsic::isVPIntrinsic(Intrinsic::ID ID) {
switch (ID) {
default:
@@ -424,10 +471,35 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const {
Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
ArrayRef<Value *> Params) {
assert(isVPIntrinsic(VPID) && "not a VP intrinsic");
-
- // TODO: Extend this for other VP intrinsics as they are upstreamed. This
- // works for binary arithmetic VP intrinsics.
- auto *VPFunc = Intrinsic::getDeclaration(M, VPID, Params[0]->getType());
+ Function *VPFunc;
+ switch (VPID) {
+ default:
+ VPFunc = Intrinsic::getDeclaration(M, VPID, Params[0]->getType());
+ break;
+ case Intrinsic::vp_load:
+ VPFunc = Intrinsic::getDeclaration(
+ M, VPID,
+ {Params[0]->getType()->getPointerElementType(), Params[0]->getType()});
+ break;
+ case Intrinsic::vp_gather:
+ VPFunc = Intrinsic::getDeclaration(
+ M, VPID,
+ {VectorType::get(cast<VectorType>(Params[0]->getType())
+ ->getElementType()
+ ->getPointerElementType(),
+ cast<VectorType>(Params[0]->getType())),
+ Params[0]->getType()});
+ break;
+ case Intrinsic::vp_store:
+ VPFunc = Intrinsic::getDeclaration(
+ M, VPID,
+ {Params[1]->getType()->getPointerElementType(), Params[1]->getType()});
+ break;
+ case Intrinsic::vp_scatter:
+ VPFunc = Intrinsic::getDeclaration(
+ M, VPID, {Params[0]->getType(), Params[1]->getType()});
+ break;
+ }
assert(VPFunc && "Could not declare VP intrinsic");
return VPFunc;
}
diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp
index 33fc647561af0..f5ff54b55cc62 100644
--- a/llvm/unittests/IR/VPIntrinsicTest.cpp
+++ b/llvm/unittests/IR/VPIntrinsicTest.cpp
@@ -46,6 +46,11 @@ class VPIntrinsicTest : public testing::Test {
Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode
<< ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) ";
+ Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, <8 x i1>, i32) ";
+ Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x i32*>, <8 x i1>, i32) ";
+ Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x i1>, i32) ";
+ Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x i1>, i32) ";
+
return parseAssemblyString(Str.str(), Err, C);
}
};
More information about the llvm-commits
mailing list