[llvm] 143f3bf - [SDAG] Split handling of VPLoad/VPGather and VPStore/VPScatter [nfc]
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 21 09:06:15 PDT 2022
Author: Philip Reames
Date: 2022-09-21T09:06:02-07:00
New Revision: 143f3bf8f422af962761b1bb1f2875a6d30b25eb
URL: https://github.com/llvm/llvm-project/commit/143f3bf8f422af962761b1bb1f2875a6d30b25eb
DIFF: https://github.com/llvm/llvm-project/commit/143f3bf8f422af962761b1bb1f2875a6d30b25eb.diff
LOG: [SDAG] Split handling of VPLoad/VPGather and VPStore/VPScatter [nfc]
The merged routines are not-idiomatic, and the code sharing that results is prettty minimal. The confusion factor is not justified.
Added:
Modified:
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 403fde7ed4a9..4bc03f70391a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7431,118 +7431,133 @@ static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
return *ResOPC;
}
-void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
- SmallVector<SDValue, 7> &OpValues,
- bool IsGather) {
+void SelectionDAGBuilder::visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT,
+ SmallVector<SDValue, 7> &OpValues) {
SDLoc DL = getCurSDLoc();
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Value *PtrOperand = VPIntrin.getArgOperand(0);
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
SDValue LD;
bool AddToChain = true;
- if (!IsGather) {
- // Do not serialize variable-length loads of constant memory with
- // anything.
- if (!Alignment)
- Alignment = DAG.getEVTAlign(VT);
- MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
- AddToChain = !AA || !AA->pointsToConstantMemory(ML);
- SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
- MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
- LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
- MMO, false /*IsExpanding */);
- } else {
- if (!Alignment)
- Alignment = DAG.getEVTAlign(VT.getScalarType());
- unsigned AS =
- PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
- MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MachinePointerInfo(AS), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
- SDValue Base, Index, Scale;
- ISD::MemIndexType IndexType;
- bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
- this, VPIntrin.getParent(),
- VT.getScalarStoreSize());
- if (!UniformBase) {
- Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
- Index = getValue(PtrOperand);
- IndexType = ISD::SIGNED_SCALED;
- Scale =
- DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
- }
- EVT IdxVT = Index.getValueType();
- EVT EltTy = IdxVT.getVectorElementType();
- if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
- EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
- Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
- }
- LD = DAG.getGatherVP(
- DAG.getVTList(VT, MVT::Other), VT, DL,
- {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
- IndexType);
- }
+ // Do not serialize variable-length loads of constant memory with
+ // anything.
+ if (!Alignment)
+ Alignment = DAG.getEVTAlign(VT);
+ MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
+ AddToChain = !AA || !AA->pointsToConstantMemory(ML);
+ SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
+ MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+ MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
+ MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
+ MMO, false /*IsExpanding */);
if (AddToChain)
PendingLoads.push_back(LD.getValue(1));
setValue(&VPIntrin, LD);
}
-void SelectionDAGBuilder::visitVPStoreScatter(const VPIntrinsic &VPIntrin,
- SmallVector<SDValue, 7> &OpValues,
- bool IsScatter) {
+void SelectionDAGBuilder::visitVPGather(const VPIntrinsic &VPIntrin, EVT VT,
+ SmallVector<SDValue, 7> &OpValues) {
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ Value *PtrOperand = VPIntrin.getArgOperand(0);
+ MaybeAlign Alignment = VPIntrin.getPointerAlignment();
+ AAMDNodes AAInfo = VPIntrin.getAAMetadata();
+ const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
+ SDValue LD;
+ if (!Alignment)
+ Alignment = DAG.getEVTAlign(VT.getScalarType());
+ unsigned AS =
+ PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
+ MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+ MachinePointerInfo(AS), MachineMemOperand::MOLoad,
+ MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ SDValue Base, Index, Scale;
+ ISD::MemIndexType IndexType;
+ bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
+ this, VPIntrin.getParent(),
+ VT.getScalarStoreSize());
+ if (!UniformBase) {
+ Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
+ Index = getValue(PtrOperand);
+ IndexType = ISD::SIGNED_SCALED;
+ Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
+ }
+ EVT IdxVT = Index.getValueType();
+ EVT EltTy = IdxVT.getVectorElementType();
+ if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
+ EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
+ Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
+ }
+ LD = DAG.getGatherVP(
+ DAG.getVTList(VT, MVT::Other), VT, DL,
+ {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
+ IndexType);
+ PendingLoads.push_back(LD.getValue(1));
+ setValue(&VPIntrin, LD);
+}
+
+void SelectionDAGBuilder::visitVPStore(const VPIntrinsic &VPIntrin,
+ SmallVector<SDValue, 7> &OpValues) {
+ SDLoc DL = getCurSDLoc();
Value *PtrOperand = VPIntrin.getArgOperand(1);
EVT VT = OpValues[0].getValueType();
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
SDValue ST;
- if (!IsScatter) {
- if (!Alignment)
- Alignment = DAG.getEVTAlign(VT);
- SDValue Ptr = OpValues[1];
- SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
- MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
- ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
- OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
- /* IsTruncating */ false, /*IsCompressing*/ false);
- } else {
- if (!Alignment)
- Alignment = DAG.getEVTAlign(VT.getScalarType());
- unsigned AS =
- PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
- MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MachinePointerInfo(AS), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
- SDValue Base, Index, Scale;
- ISD::MemIndexType IndexType;
- bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
- this, VPIntrin.getParent(),
- VT.getScalarStoreSize());
- if (!UniformBase) {
- Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
- Index = getValue(PtrOperand);
- IndexType = ISD::SIGNED_SCALED;
- Scale =
- DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
- }
- EVT IdxVT = Index.getValueType();
- EVT EltTy = IdxVT.getVectorElementType();
- if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
- EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
- Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
- }
- ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
- {getMemoryRoot(), OpValues[0], Base, Index, Scale,
- OpValues[2], OpValues[3]},
- MMO, IndexType);
+ if (!Alignment)
+ Alignment = DAG.getEVTAlign(VT);
+ SDValue Ptr = OpValues[1];
+ SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
+ MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+ MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
+ MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
+ OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
+ /* IsTruncating */ false, /*IsCompressing*/ false);
+ DAG.setRoot(ST);
+ setValue(&VPIntrin, ST);
+}
+
+void SelectionDAGBuilder::visitVPScatter(const VPIntrinsic &VPIntrin,
+ SmallVector<SDValue, 7> &OpValues) {
+ SDLoc DL = getCurSDLoc();
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ Value *PtrOperand = VPIntrin.getArgOperand(1);
+ EVT VT = OpValues[0].getValueType();
+ MaybeAlign Alignment = VPIntrin.getPointerAlignment();
+ AAMDNodes AAInfo = VPIntrin.getAAMetadata();
+ SDValue ST;
+ if (!Alignment)
+ Alignment = DAG.getEVTAlign(VT.getScalarType());
+ unsigned AS =
+ PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
+ MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+ MachinePointerInfo(AS), MachineMemOperand::MOStore,
+ MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ SDValue Base, Index, Scale;
+ ISD::MemIndexType IndexType;
+ bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
+ this, VPIntrin.getParent(),
+ VT.getScalarStoreSize());
+ if (!UniformBase) {
+ Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
+ Index = getValue(PtrOperand);
+ IndexType = ISD::SIGNED_SCALED;
+ Scale =
+ DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
+ }
+ EVT IdxVT = Index.getValueType();
+ EVT EltTy = IdxVT.getVectorElementType();
+ if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
+ EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
+ Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
}
+ ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
+ {getMemoryRoot(), OpValues[0], Base, Index, Scale,
+ OpValues[2], OpValues[3]},
+ MMO, IndexType);
DAG.setRoot(ST);
setValue(&VPIntrin, ST);
}
@@ -7669,16 +7684,19 @@ void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
break;
}
case ISD::VP_LOAD:
+ visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
+ break;
case ISD::VP_GATHER:
- visitVPLoadGather(VPIntrin, ValueVTs[0], OpValues,
- Opcode == ISD::VP_GATHER);
+ visitVPGather(VPIntrin, ValueVTs[0], OpValues);
break;
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
break;
case ISD::VP_STORE:
+ visitVPStore(VPIntrin, OpValues);
+ break;
case ISD::VP_SCATTER:
- visitVPStoreScatter(VPIntrin, OpValues, Opcode == ISD::VP_SCATTER);
+ visitVPScatter(VPIntrin, OpValues);
break;
case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
visitVPStridedStore(VPIntrin, OpValues);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 6838676edbf4..f53ceacc96c3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -569,10 +569,14 @@ class SelectionDAGBuilder {
void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
- void visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
- SmallVector<SDValue, 7> &OpValues, bool IsGather);
- void visitVPStoreScatter(const VPIntrinsic &VPIntrin,
- SmallVector<SDValue, 7> &OpValues, bool IsScatter);
+ void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT,
+ SmallVector<SDValue, 7> &OpValues);
+ void visitVPStore(const VPIntrinsic &VPIntrin,
+ SmallVector<SDValue, 7> &OpValues);
+ void visitVPGather(const VPIntrinsic &VPIntrin, EVT VT,
+ SmallVector<SDValue, 7> &OpValues);
+ void visitVPScatter(const VPIntrinsic &VPIntrin,
+ SmallVector<SDValue, 7> &OpValues);
void visitVPStridedLoad(const VPIntrinsic &VPIntrin, EVT VT,
SmallVectorImpl<SDValue> &OpValues);
void visitVPStridedStore(const VPIntrinsic &VPIntrin,
More information about the llvm-commits
mailing list